Compare commits
28 Commits
a57a6b14d4
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
dd48ef9194 | ||
|
|
a888acf0b7 | ||
|
|
b66ee3095d | ||
|
|
525bca255b | ||
|
|
32a3f876e3 | ||
|
|
ccf815b71f | ||
|
|
da6b54079f | ||
|
|
43d58ff093 | ||
|
|
677f72df33 | ||
|
|
18d6832f95 | ||
|
|
170c42c282 | ||
|
|
51eafc9a07 | ||
|
|
01213afd8b | ||
|
|
2ef1ef80e4 | ||
|
|
5ed7c611e9 | ||
|
|
f7f149d091 | ||
|
|
b858786101 | ||
|
|
c9a24069da | ||
|
|
99c7407894 | ||
|
|
8a8574e61e | ||
|
|
ee10188923 | ||
|
|
da7c9d4881 | ||
|
|
48f8c8da66 | ||
|
|
183f87155f | ||
|
|
7990f359d3 | ||
|
|
eedf168476 | ||
|
|
d415e2c4f9 | ||
|
|
e0b00b7776 |
2
common
2
common
Submodule common updated: 1374f90397...ec0812ad9d
@@ -199,6 +199,12 @@ module.exports = {
|
||||
// source: 'http://nsfw.unknown.name/random',
|
||||
},
|
||||
},
|
||||
webApi: {
|
||||
enabled: true,
|
||||
address: 'http://localhost:5100/api',
|
||||
apiUserId: 1,
|
||||
apiKey: null,
|
||||
},
|
||||
proxy: {
|
||||
enable: false,
|
||||
test: 'https://api.ipify.org?format=json',
|
||||
@@ -280,6 +286,9 @@ module.exports = {
|
||||
excludeHostnames: [],
|
||||
selectIndex: {},
|
||||
},
|
||||
biometrics: {
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
titleSlugLength: 50,
|
||||
};
|
||||
|
||||
@@ -76,7 +76,7 @@ exports.up = async (knex) => {
|
||||
});
|
||||
|
||||
await knex.schema.createTable('media', (table) => {
|
||||
table.text('id', 21)
|
||||
table.string('id', 21)
|
||||
.primary();
|
||||
|
||||
table.text('path');
|
||||
|
||||
87
migrations/20260608053154_sync_abilities.js
Normal file
87
migrations/20260608053154_sync_abilities.js
Normal file
@@ -0,0 +1,87 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.createTable('sync', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.string('domain');
|
||||
table.specificType('item_ids', 'integer array');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ subject: 'sync' },
|
||||
{ subject: 'plainUrls' },
|
||||
]))
|
||||
.where('role', 'admin');
|
||||
|
||||
await knex.raw(`
|
||||
DROP TABLE IF EXISTS releases_search CASCADE;
|
||||
DROP TABLE IF EXISTS movies_search CASCADE;
|
||||
DROP TABLE IF EXISTS series_search CASCADE;
|
||||
|
||||
DROP TABLE IF EXISTS releases_search_results CASCADE;
|
||||
DROP TABLE IF EXISTS movies_search_results CASCADE;
|
||||
`);
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.dropTable('sync');
|
||||
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ plainUrls: true },
|
||||
]))
|
||||
.where('role', 'admin');
|
||||
|
||||
await knex.schema.createTable('releases_search', (table) => {
|
||||
table.integer('release_id', 16)
|
||||
.references('id')
|
||||
.inTable('releases')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('movies_search', (table) => {
|
||||
table.integer('movie_id', 16)
|
||||
.references('id')
|
||||
.inTable('movies')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('series_search', (table) => {
|
||||
table.integer('serie_id', 16)
|
||||
.references('id')
|
||||
.inTable('series')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.raw(`
|
||||
ALTER TABLE releases_search ADD COLUMN document tsvector;
|
||||
ALTER TABLE movies_search ADD COLUMN document tsvector;
|
||||
ALTER TABLE series_search ADD COLUMN document tsvector;
|
||||
|
||||
CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
|
||||
CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
|
||||
CREATE INDEX releases_search_index ON releases_search USING GIN (document);
|
||||
CREATE INDEX movies_search_index ON movies_search USING GIN (document);
|
||||
CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
|
||||
CREATE INDEX series_search_index ON series_search USING GIN (document);
|
||||
`);
|
||||
};
|
||||
35
migrations/20260704023524_biometrics.js
Normal file
35
migrations/20260704023524_biometrics.js
Normal file
@@ -0,0 +1,35 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.createTable('media_biometrics', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.string('media_id', 21)
|
||||
.references('id')
|
||||
.inTable('media')
|
||||
.notNullable()
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('width');
|
||||
table.integer('height');
|
||||
|
||||
table.integer('face_index')
|
||||
.notNullable()
|
||||
.defaultTo(0);
|
||||
|
||||
table.json('biometrics');
|
||||
table.specificType('embedding', 'vector(1024)');
|
||||
|
||||
table.datetime('updated_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
|
||||
table.unique(['media_id', 'face_index']);
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.dropTable('media_biometrics');
|
||||
};
|
||||
659
package-lock.json
generated
659
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.252.18",
|
||||
"version": "1.254.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "traxxx",
|
||||
"version": "1.252.18",
|
||||
"version": "1.254.1",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.458.0",
|
||||
@@ -17,6 +17,8 @@
|
||||
"@graphile-contrib/pg-order-by-related": "^1.0.0",
|
||||
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
|
||||
"@graphile/pg-aggregates": "^0.1.1",
|
||||
"@tensorflow/tfjs-node": "^4.22.0",
|
||||
"@vladmandic/human": "^3.3.6",
|
||||
"acorn": "^8.11.2",
|
||||
"array-equal": "^1.0.2",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
@@ -25,7 +27,6 @@
|
||||
"bluebird": "^3.7.2",
|
||||
"body-parser": "^1.20.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
"canvas": "^2.11.2",
|
||||
"casual": "^1.6.2",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"cli-confirm": "^1.0.1",
|
||||
@@ -3811,6 +3812,8 @@
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
|
||||
"integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"detect-libc": "^2.0.0",
|
||||
"https-proxy-agent": "^5.0.0",
|
||||
@@ -3830,6 +3833,8 @@
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
|
||||
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
@@ -3842,6 +3847,8 @@
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
|
||||
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"yallist": "^4.0.0"
|
||||
},
|
||||
@@ -3853,6 +3860,8 @@
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
|
||||
"integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"semver": "^6.0.0"
|
||||
},
|
||||
@@ -3867,6 +3876,8 @@
|
||||
"version": "6.3.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
|
||||
"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
}
|
||||
@@ -3875,6 +3886,8 @@
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
@@ -3894,6 +3907,8 @@
|
||||
"version": "7.5.4",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
|
||||
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"lru-cache": "^6.0.0"
|
||||
},
|
||||
@@ -3907,17 +3922,23 @@
|
||||
"node_modules/@mapbox/node-pre-gyp/node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||
"optional": true,
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/@mapbox/node-pre-gyp/node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
|
||||
"optional": true,
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/@mapbox/node-pre-gyp/node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
@@ -3926,7 +3947,9 @@
|
||||
"node_modules/@mapbox/node-pre-gyp/node_modules/yallist": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
|
||||
"optional": true,
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/@nicolo-ribaudo/chokidar-2": {
|
||||
"version": "2.1.8-no-fsevents.3",
|
||||
@@ -5051,6 +5074,27 @@
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
|
||||
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-4.22.0.tgz",
|
||||
"integrity": "sha512-0TrIrXs6/b7FLhLVNmfh8Sah6JgjBPH4mZ8JGb7NU6WW+cx00qK5BcAZxw7NCzxj6N8MRAIfHq+oNbPUNG5VAg==",
|
||||
"dependencies": {
|
||||
"@tensorflow/tfjs-backend-cpu": "4.22.0",
|
||||
"@tensorflow/tfjs-backend-webgl": "4.22.0",
|
||||
"@tensorflow/tfjs-converter": "4.22.0",
|
||||
"@tensorflow/tfjs-core": "4.22.0",
|
||||
"@tensorflow/tfjs-data": "4.22.0",
|
||||
"@tensorflow/tfjs-layers": "4.22.0",
|
||||
"argparse": "^1.0.10",
|
||||
"chalk": "^4.1.0",
|
||||
"core-js": "3.29.1",
|
||||
"regenerator-runtime": "^0.13.5",
|
||||
"yargs": "^16.0.3"
|
||||
},
|
||||
"bin": {
|
||||
"tfjs-custom-module": "dist/tools/custom_module/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-core": {
|
||||
"version": "1.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.7.0.tgz",
|
||||
@@ -5075,6 +5119,493 @@
|
||||
"node": "4.x || >=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-node/-/tfjs-node-4.22.0.tgz",
|
||||
"integrity": "sha512-uHrXeUlfgkMxTZqHkESSV7zSdKdV0LlsBeblqkuKU9nnfxB1pC6DtoyYVaLxznzZy7WQSegjcohxxCjAf6Dc7w==",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"@mapbox/node-pre-gyp": "1.0.9",
|
||||
"@tensorflow/tfjs": "4.22.0",
|
||||
"adm-zip": "^0.5.2",
|
||||
"google-protobuf": "^3.9.2",
|
||||
"https-proxy-agent": "^2.2.1",
|
||||
"progress": "^2.0.0",
|
||||
"rimraf": "^2.6.2",
|
||||
"tar": "^6.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.11.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.9.tgz",
|
||||
"integrity": "sha512-aDF3S3rK9Q2gey/WAttUlISduDItz5BU3306M9Eyv6/oS40aMprnopshtlKTykxRNIBEZuRMaZAnbrQ4QtKGyw==",
|
||||
"dependencies": {
|
||||
"detect-libc": "^2.0.0",
|
||||
"https-proxy-agent": "^5.0.0",
|
||||
"make-dir": "^3.1.0",
|
||||
"node-fetch": "^2.6.7",
|
||||
"nopt": "^5.0.0",
|
||||
"npmlog": "^5.0.1",
|
||||
"rimraf": "^3.0.2",
|
||||
"semver": "^7.3.5",
|
||||
"tar": "^6.1.11"
|
||||
},
|
||||
"bin": {
|
||||
"node-pre-gyp": "bin/node-pre-gyp"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp/node_modules/https-proxy-agent": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
|
||||
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
|
||||
"dependencies": {
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp/node_modules/minipass": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
|
||||
"integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp/node_modules/mkdirp": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
|
||||
"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
|
||||
"bin": {
|
||||
"mkdirp": "bin/cmd.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp/node_modules/rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
|
||||
"deprecated": "Rimraf versions prior to v4 are no longer supported",
|
||||
"dependencies": {
|
||||
"glob": "^7.1.3"
|
||||
},
|
||||
"bin": {
|
||||
"rimraf": "bin.js"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp/node_modules/semver": {
|
||||
"version": "7.8.5",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz",
|
||||
"integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==",
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp/node_modules/tar": {
|
||||
"version": "6.2.1",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz",
|
||||
"integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==",
|
||||
"deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me",
|
||||
"dependencies": {
|
||||
"chownr": "^2.0.0",
|
||||
"fs-minipass": "^2.0.0",
|
||||
"minipass": "^5.0.0",
|
||||
"minizlib": "^2.1.1",
|
||||
"mkdirp": "^1.0.3",
|
||||
"yallist": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/@mapbox/node-pre-gyp/node_modules/yallist": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/https-proxy-agent": {
|
||||
"version": "2.2.4",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz",
|
||||
"integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==",
|
||||
"dependencies": {
|
||||
"agent-base": "^4.3.0",
|
||||
"debug": "^3.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 4.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/https-proxy-agent/node_modules/agent-base": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz",
|
||||
"integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==",
|
||||
"dependencies": {
|
||||
"es6-promisify": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/https-proxy-agent/node_modules/debug": {
|
||||
"version": "3.2.7",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
|
||||
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/make-dir": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
|
||||
"integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==",
|
||||
"dependencies": {
|
||||
"semver": "^6.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/rimraf": {
|
||||
"version": "2.7.1",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
|
||||
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
|
||||
"deprecated": "Rimraf versions prior to v4 are no longer supported",
|
||||
"dependencies": {
|
||||
"glob": "^7.1.3"
|
||||
},
|
||||
"bin": {
|
||||
"rimraf": "bin.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-node/node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-cpu": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.22.0.tgz",
|
||||
"integrity": "sha512-1u0FmuLGuRAi8D2c3cocHTASGXOmHc/4OvoVDENJayjYkS119fcTcQf4iHrtLthWyDIPy3JiPhRrZQC9EwnhLw==",
|
||||
"dependencies": {
|
||||
"@types/seedrandom": "^2.4.28",
|
||||
"seedrandom": "^3.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"yarn": ">= 1.3.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@tensorflow/tfjs-core": "4.22.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-backend-webgl": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.22.0.tgz",
|
||||
"integrity": "sha512-H535XtZWnWgNwSzv538czjVlbJebDl5QTMOth4RXr2p/kJ1qSIXE0vZvEtO+5EC9b00SvhplECny2yDewQb/Yg==",
|
||||
"dependencies": {
|
||||
"@tensorflow/tfjs-backend-cpu": "4.22.0",
|
||||
"@types/offscreencanvas": "~2019.3.0",
|
||||
"@types/seedrandom": "^2.4.28",
|
||||
"seedrandom": "^3.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"yarn": ">= 1.3.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@tensorflow/tfjs-core": "4.22.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-converter": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.22.0.tgz",
|
||||
"integrity": "sha512-PT43MGlnzIo+YfbsjM79Lxk9lOq6uUwZuCc8rrp0hfpLjF6Jv8jS84u2jFb+WpUeuF4K33ZDNx8CjiYrGQ2trQ==",
|
||||
"peerDependencies": {
|
||||
"@tensorflow/tfjs-core": "4.22.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.22.0.tgz",
|
||||
"integrity": "sha512-LEkOyzbknKFoWUwfkr59vSB68DMJ4cjwwHgicXN0DUi3a0Vh1Er3JQqCI1Hl86GGZQvY8ezVrtDIvqR1ZFW55A==",
|
||||
"dependencies": {
|
||||
"@types/long": "^4.0.1",
|
||||
"@types/offscreencanvas": "~2019.7.0",
|
||||
"@types/seedrandom": "^2.4.28",
|
||||
"@webgpu/types": "0.1.38",
|
||||
"long": "4.0.0",
|
||||
"node-fetch": "~2.6.1",
|
||||
"seedrandom": "^3.0.5"
|
||||
},
|
||||
"engines": {
|
||||
"yarn": ">= 1.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-core/node_modules/@types/offscreencanvas": {
|
||||
"version": "2019.7.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz",
|
||||
"integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-data": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-4.22.0.tgz",
|
||||
"integrity": "sha512-dYmF3LihQIGvtgJrt382hSRH4S0QuAp2w1hXJI2+kOaEqo5HnUPG0k5KA6va+S1yUhx7UBToUKCBHeLHFQRV4w==",
|
||||
"dependencies": {
|
||||
"@types/node-fetch": "^2.1.2",
|
||||
"node-fetch": "~2.6.1",
|
||||
"string_decoder": "^1.3.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@tensorflow/tfjs-core": "4.22.0",
|
||||
"seedrandom": "^3.0.5"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@tensorflow/tfjs-layers": {
|
||||
"version": "4.22.0",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-4.22.0.tgz",
|
||||
"integrity": "sha512-lybPj4ZNj9iIAPUj7a8ZW1hg8KQGfqWLlCZDi9eM/oNKCCAgchiyzx8OrYoWmRrB+AM6VNEeIT+2gZKg5ReihA==",
|
||||
"peerDependencies": {
|
||||
"@tensorflow/tfjs-core": "4.22.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/@types/seedrandom": {
|
||||
"version": "2.4.34",
|
||||
"resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.34.tgz",
|
||||
"integrity": "sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/ansi-styles": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"dependencies": {
|
||||
"color-convert": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/argparse": {
|
||||
"version": "1.0.10",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
|
||||
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
|
||||
"dependencies": {
|
||||
"sprintf-js": "~1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/chalk": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
|
||||
"dependencies": {
|
||||
"ansi-styles": "^4.1.0",
|
||||
"supports-color": "^7.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/cliui": {
|
||||
"version": "7.0.4",
|
||||
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
|
||||
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
|
||||
"dependencies": {
|
||||
"string-width": "^4.2.0",
|
||||
"strip-ansi": "^6.0.0",
|
||||
"wrap-ansi": "^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"dependencies": {
|
||||
"color-name": "~1.1.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/core-js": {
|
||||
"version": "3.29.1",
|
||||
"resolved": "https://registry.npmjs.org/core-js/-/core-js-3.29.1.tgz",
|
||||
"integrity": "sha512-+jwgnhg6cQxKYIIjGtAHq2nwUOolo9eoFZ4sHfUH09BLXBgxnH4gA0zEd+t+BO2cNB8idaBtZFcFTRjQJRJmAw==",
|
||||
"hasInstallScript": true,
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/core-js"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/node-fetch": {
|
||||
"version": "2.6.13",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.13.tgz",
|
||||
"integrity": "sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/regenerator-runtime": {
|
||||
"version": "0.13.11",
|
||||
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
|
||||
"integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/seedrandom": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
|
||||
"integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
|
||||
"dependencies": {
|
||||
"safe-buffer": "~5.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dependencies": {
|
||||
"has-flag": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/wrap-ansi": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
|
||||
"dependencies": {
|
||||
"ansi-styles": "^4.0.0",
|
||||
"string-width": "^4.1.0",
|
||||
"strip-ansi": "^6.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/node_modules/yargs": {
|
||||
"version": "16.2.2",
|
||||
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.2.tgz",
|
||||
"integrity": "sha512-Nt9ZJjXTv5R8MHbqby/wXQ6Gi0Bb3TcYZkR1bzuL4yB2OxWPkXknz513gEF0GoA6tn00UpbPvERW8rzCuWCA6w==",
|
||||
"dependencies": {
|
||||
"cliui": "^7.0.2",
|
||||
"escalade": "^3.1.1",
|
||||
"get-caller-file": "^2.0.5",
|
||||
"require-directory": "^2.1.1",
|
||||
"string-width": "^4.2.0",
|
||||
"y18n": "^5.0.5",
|
||||
"yargs-parser": "^20.2.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/@tokenizer/token": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz",
|
||||
@@ -5180,6 +5711,11 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/long": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
|
||||
"integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA=="
|
||||
},
|
||||
"node_modules/@types/minimist": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz",
|
||||
@@ -5199,6 +5735,15 @@
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node-fetch": {
|
||||
"version": "2.6.13",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz",
|
||||
"integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^4.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/normalize-package-data": {
|
||||
"version": "2.4.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz",
|
||||
@@ -5446,6 +5991,15 @@
|
||||
"is-function": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@vladmandic/human": {
|
||||
"version": "3.3.6",
|
||||
"resolved": "https://registry.npmjs.org/@vladmandic/human/-/human-3.3.6.tgz",
|
||||
"integrity": "sha512-Nr5mPfq1gQ+uKeXY5uM3Fj8UxvF5CEh2s6txM5wRThqaW0mF9huooOuZSrT8hhGho0hGaXLFdAwfD/2+teCv6A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@vue/compiler-core": {
|
||||
"version": "3.5.25",
|
||||
"resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.25.tgz",
|
||||
@@ -5688,6 +6242,11 @@
|
||||
"@xtuc/long": "4.2.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@webgpu/types": {
|
||||
"version": "0.1.38",
|
||||
"resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.38.tgz",
|
||||
"integrity": "sha512-7LrhVKz2PRh+DD7+S+PVaFd5HxaWQvoMqBbsV9fNJO1pjUs1P8bM2vQVNfk+3URTqbuTI7gkXi0rfsN0IadoBA=="
|
||||
},
|
||||
"node_modules/@webpack-cli/configtest": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-2.1.1.tgz",
|
||||
@@ -5831,6 +6390,14 @@
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/adm-zip": {
|
||||
"version": "0.5.18",
|
||||
"resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz",
|
||||
"integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==",
|
||||
"engines": {
|
||||
"node": ">=12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/aes-decrypter": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/aes-decrypter/-/aes-decrypter-4.0.1.tgz",
|
||||
@@ -7209,6 +7776,8 @@
|
||||
"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz",
|
||||
"integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==",
|
||||
"hasInstallScript": true,
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@mapbox/node-pre-gyp": "^1.0.0",
|
||||
"nan": "^2.17.0",
|
||||
@@ -8249,6 +8818,8 @@
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
|
||||
"integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"mimic-response": "^2.0.0"
|
||||
},
|
||||
@@ -8815,13 +9386,14 @@
|
||||
}
|
||||
},
|
||||
"node_modules/es-set-tostringtag": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.2.tgz",
|
||||
"integrity": "sha512-BuDyupZt65P9D2D2vA/zqcI3G5xRsklm5N3xCwuiy+/vKy8i0ifdsQP1sLgO4tZDSCaQUSnmC48khknGMV3D2Q==",
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
|
||||
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
|
||||
"dependencies": {
|
||||
"get-intrinsic": "^1.2.2",
|
||||
"has-tostringtag": "^1.0.0",
|
||||
"hasown": "^2.0.0"
|
||||
"es-errors": "^1.3.0",
|
||||
"get-intrinsic": "^1.2.6",
|
||||
"has-tostringtag": "^1.0.2",
|
||||
"hasown": "^2.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
@@ -8851,6 +9423,19 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/es6-promise": {
|
||||
"version": "4.2.8",
|
||||
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
|
||||
"integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w=="
|
||||
},
|
||||
"node_modules/es6-promisify": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
|
||||
"integrity": "sha512-C+d6UdsYDk0lMebHNR4S2NybQMMngAOnOwYBQjTOiv0MkoJMP0Myw2mgpDLBcpfCmRLxyFqYhS/CfOENq4SJhQ==",
|
||||
"dependencies": {
|
||||
"es6-promise": "^4.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/escalade": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
||||
@@ -10250,13 +10835,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/form-data": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
|
||||
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz",
|
||||
"integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
"es-set-tostringtag": "^2.1.0",
|
||||
"hasown": "^2.0.4",
|
||||
"mime-types": "^2.1.35"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
@@ -10733,6 +11320,11 @@
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/google-protobuf": {
|
||||
"version": "3.21.4",
|
||||
"resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.21.4.tgz",
|
||||
"integrity": "sha512-MnG7N936zcKTco4Jd2PX2U96Kf9PxygAPKBug+74LHzmHXmceN16MmRcdgZv+DGef/S9YvQAfRsNCn4cjf9yyQ=="
|
||||
},
|
||||
"node_modules/gopd": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
|
||||
@@ -10984,11 +11576,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/has-tostringtag": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
|
||||
"integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
|
||||
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
|
||||
"dependencies": {
|
||||
"has-symbols": "^1.0.2"
|
||||
"has-symbols": "^1.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
@@ -11009,9 +11601,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/hasown": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
|
||||
"integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz",
|
||||
"integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==",
|
||||
"dependencies": {
|
||||
"function-bind": "^1.1.2"
|
||||
},
|
||||
@@ -13030,6 +13622,11 @@
|
||||
"node": ">= 12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/long": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
|
||||
"integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="
|
||||
},
|
||||
"node_modules/longjohn": {
|
||||
"version": "0.2.12",
|
||||
"resolved": "https://registry.npmjs.org/longjohn/-/longjohn-0.2.12.tgz",
|
||||
@@ -13422,6 +14019,8 @@
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
|
||||
"integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
@@ -19475,12 +20074,16 @@
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
]
|
||||
],
|
||||
"optional": true,
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/simple-get": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz",
|
||||
"integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"decompress-response": "^4.2.0",
|
||||
"once": "^1.3.1",
|
||||
@@ -20130,9 +20733,10 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tar": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz",
|
||||
"integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==",
|
||||
"version": "6.2.1",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz",
|
||||
"integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==",
|
||||
"deprecated": "Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me",
|
||||
"dependencies": {
|
||||
"chownr": "^2.0.0",
|
||||
"fs-minipass": "^2.0.0",
|
||||
@@ -22729,7 +23333,6 @@
|
||||
"version": "20.2.9",
|
||||
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz",
|
||||
"integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.252.18",
|
||||
"version": "1.254.1",
|
||||
"description": "All the latest porn releases in one place",
|
||||
"main": "src/app.js",
|
||||
"scripts": {
|
||||
@@ -76,6 +76,8 @@
|
||||
"@graphile-contrib/pg-order-by-related": "^1.0.0",
|
||||
"@graphile-contrib/pg-simplify-inflector": "^6.1.0",
|
||||
"@graphile/pg-aggregates": "^0.1.1",
|
||||
"@tensorflow/tfjs-node": "^4.22.0",
|
||||
"@vladmandic/human": "^3.3.6",
|
||||
"acorn": "^8.11.2",
|
||||
"array-equal": "^1.0.2",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
@@ -84,7 +86,6 @@
|
||||
"bluebird": "^3.7.2",
|
||||
"body-parser": "^1.20.2",
|
||||
"bottleneck": "^2.19.5",
|
||||
"canvas": "^2.11.2",
|
||||
"casual": "^1.6.2",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"cli-confirm": "^1.0.1",
|
||||
|
||||
@@ -647,6 +647,7 @@ const tags = [
|
||||
{
|
||||
name: 'MFF threesome',
|
||||
slug: 'mff',
|
||||
implies: ['threesome'],
|
||||
description: 'A threesome with two women and one guy, in which the women have sex with eachother.',
|
||||
group: 'group',
|
||||
},
|
||||
@@ -830,6 +831,7 @@ const tags = [
|
||||
{
|
||||
name: 'MFM threesome',
|
||||
slug: 'mfm',
|
||||
implies: ['threesome'],
|
||||
description: 'Two men fucking one woman, but not eachother. Typically involves a \'spitroast\', where one guy gets a blowjob and the other fucks her pussy or ass.',
|
||||
group: 'group',
|
||||
},
|
||||
@@ -3084,9 +3086,18 @@ const aliases = [
|
||||
name: 'stepmom',
|
||||
for: 'family',
|
||||
},
|
||||
{
|
||||
name: 'ass2mouth',
|
||||
for: 'atm',
|
||||
},
|
||||
{
|
||||
name: 'fist',
|
||||
for: 'fisting',
|
||||
},
|
||||
];
|
||||
|
||||
const priorities = [ // higher index is higher priority
|
||||
['blonde', 'brunette', 'black-hair', 'redhead'],
|
||||
['double-dildo', 'double-dildo-blowjob', 'double-dildo-kiss', 'double-dildo-anal', 'double-dildo-dp'],
|
||||
['toys', 'toy-anal', 'toy-dp', 'piss-drinking'],
|
||||
['family'],
|
||||
|
||||
28
seeds/08_abilities.js
Normal file
28
seeds/08_abilities.js
Normal file
@@ -0,0 +1,28 @@
|
||||
exports.seed = async (knex) => {
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ subject: 'sync' },
|
||||
{ subject: 'plainUrls' },
|
||||
]))
|
||||
.where('role', 'admin');
|
||||
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ subject: 'plainUrls' },
|
||||
]))
|
||||
.where('role', 'editor');
|
||||
};
|
||||
@@ -30,6 +30,8 @@ const { toBaseReleases } = require('./deep');
|
||||
const { associateAvatars, flushOrphanedMedia } = require('./media');
|
||||
const { fetchEntitiesBySlug } = require('./entities');
|
||||
const { deleteScenes } = require('./releases');
|
||||
const { updateActorSearch } = require('./update-search');
|
||||
const { setBiometrics } = require('./biometrics');
|
||||
|
||||
const actorsCommon = import('../common/actors.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
|
||||
const geoCommon = import('../common/geo.mjs'); // eslint-disable-line import/extensions, import/no-relative-packages
|
||||
@@ -823,6 +825,11 @@ async function storeProfiles(profiles) {
|
||||
|
||||
await upsertProfiles(profilesWithAvatarIds);
|
||||
await interpolateProfiles(actorIds);
|
||||
|
||||
setBiometrics(Array.from(new Set(profiles.map((profile) => profile.actorId)))).catch((error) => {
|
||||
// biometrics are courtesy, don't await
|
||||
logger.error(`Biometrics failed: ${error.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeActors(argNames) {
|
||||
@@ -912,7 +919,7 @@ async function getOrCreateActors(baseActors, batchId) {
|
||||
.whereRaw(`
|
||||
actors.slug = base_actors.slug
|
||||
AND actors.entity_id IS NULL
|
||||
AND NOT base_actors.collision_likely
|
||||
AND (NOT base_actors.collision_likely OR actors.allow_global_match)
|
||||
`)
|
||||
.orWhereRaw(`
|
||||
actors.slug = base_actors.slug
|
||||
@@ -980,7 +987,8 @@ async function getOrCreateActors(baseActors, batchId) {
|
||||
|
||||
await storeProfiles(newActorProfiles);
|
||||
|
||||
if (Array.isArray(newActors)) {
|
||||
if (Array.isArray(newActors) && newActors.length > 0) {
|
||||
await updateActorSearch(newActors.map((actor) => actor.id));
|
||||
return newActors.concat(existingActors);
|
||||
}
|
||||
|
||||
|
||||
204
src/biometrics.js
Normal file
204
src/biometrics.js
Normal file
@@ -0,0 +1,204 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const path = require('path');
|
||||
const fsPromises = require('fs').promises;
|
||||
const Human = require('@vladmandic/human').default;
|
||||
|
||||
const knex = require('./knex');
|
||||
const logger = require('./logger')(__filename);
|
||||
|
||||
const humanConfig = {
|
||||
modelBasePath: `file://${path.join(__dirname, '../node_modules/@vladmandic/human/models')}/`,
|
||||
backend: 'tensorflow', // uses tfjs-node under the hood
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { rotation: false },
|
||||
mesh: { enabled: false },
|
||||
iris: { enabled: false },
|
||||
description: { enabled: true },
|
||||
emotion: { enabled: true },
|
||||
},
|
||||
body: { enabled: false },
|
||||
hand: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
};
|
||||
|
||||
let humanInstance = null;
|
||||
const nsPerSec = 1e9;
|
||||
|
||||
async function getHumanInstance() {
|
||||
if (!humanInstance) {
|
||||
humanInstance = new Human(humanConfig);
|
||||
|
||||
await humanInstance.load();
|
||||
await humanInstance.warmup();
|
||||
}
|
||||
|
||||
return humanInstance;
|
||||
}
|
||||
|
||||
async function getImageBuffer(mediaEntry) {
|
||||
if (mediaEntry.is_s3) {
|
||||
if (!config.s3.enabled) {
|
||||
logger.verbose(`Skipping biometrics for ${mediaEntry.media_id}, S3 not enabled`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const res = await fetch(`https://${config.s3.bucket}/${mediaEntry.path}`);
|
||||
|
||||
if (res.ok) {
|
||||
return Buffer.from(await res.arrayBuffer());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const buffer = await fsPromises.readFile(path.join(config.media.path, mediaEntry.path));
|
||||
|
||||
return buffer;
|
||||
} catch (_error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function getBiometrics(avatarEntry) {
|
||||
if (!avatarEntry) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const human = await getHumanInstance();
|
||||
const buffer = await getImageBuffer(avatarEntry);
|
||||
|
||||
if (!buffer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const tensor = human.tf.node.decodeImage(buffer, 3);
|
||||
|
||||
try {
|
||||
const result = await human.detect(tensor);
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
|
||||
return null;
|
||||
} finally {
|
||||
tensor.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
function getAvatarSource(actorIds, includePhotos) {
|
||||
if (includePhotos) {
|
||||
return knex('actors_avatars')
|
||||
.select('actor_id', 'media_id')
|
||||
.modify((builder) => {
|
||||
if (actorIds) {
|
||||
builder.whereIn('actor_id', actorIds);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return knex('actors')
|
||||
.select('id as actor_id', 'avatar_media_id as media_id')
|
||||
.whereNotNull('avatar_media_id')
|
||||
.modify((builder) => {
|
||||
if (actorIds) {
|
||||
builder.whereIn('id', actorIds);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function setBiometrics(actorIds, shouldUpdate = false, includePhotos = false) {
|
||||
if (!config.media.biometrics.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const startTime = process.hrtime();
|
||||
|
||||
const avatarSource = getAvatarSource(actorIds, includePhotos);
|
||||
|
||||
const avatarEntries = await knex
|
||||
.select(
|
||||
'avatar_source.actor_id',
|
||||
'media.id as media_id',
|
||||
'media.path',
|
||||
'media.is_s3',
|
||||
knex.raw('coalesce(json_agg(media_biometrics.biometrics) filter (where media_biometrics.id is not null), \'[]\') as biometrics'),
|
||||
)
|
||||
.from(avatarSource.as('avatar_source'))
|
||||
.leftJoin('media', 'media.id', 'avatar_source.media_id')
|
||||
.leftJoin('media_biometrics', 'media_biometrics.media_id', 'media.id')
|
||||
.groupBy('avatar_source.actor_id', 'media.id', 'media.path', 'media.is_s3');
|
||||
|
||||
await avatarEntries.reduce(async (chain, avatarEntry, avatarIndex) => {
|
||||
await chain;
|
||||
|
||||
if (avatarEntry.biometrics.length > 0 && !shouldUpdate) {
|
||||
logger.verbose(`Skipping biometrics for ${avatarEntry.media_id}, already present`);
|
||||
return;
|
||||
}
|
||||
|
||||
const biometrics = await getBiometrics(avatarEntry);
|
||||
|
||||
if (!biometrics) {
|
||||
return;
|
||||
}
|
||||
|
||||
const curatedBiometrics = biometrics.face.map((face) => ({
|
||||
biometrics: {
|
||||
age: face.age,
|
||||
gender: face.gender,
|
||||
genderScore: face.genderScore,
|
||||
emotion: face.emotion,
|
||||
box: face.box,
|
||||
score: face.score,
|
||||
...Object.fromEntries(Object.entries(face.annotations).map(([key, annotation]) => [key, annotation.filter(Boolean)[0] || null])),
|
||||
},
|
||||
embedding: face.embedding,
|
||||
}));
|
||||
|
||||
if (curatedBiometrics.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (shouldUpdate) {
|
||||
await knex('media_biometrics')
|
||||
.where('media_id', avatarEntry.media_id)
|
||||
.delete();
|
||||
}
|
||||
|
||||
await knex('media_biometrics')
|
||||
.insert(curatedBiometrics.map((face, index) => ({
|
||||
media_id: avatarEntry.media_id,
|
||||
face_index: index,
|
||||
width: biometrics.width,
|
||||
height: biometrics.height,
|
||||
biometrics: JSON.stringify(face.biometrics),
|
||||
embedding: knex.raw('?::vector', [JSON.stringify(face.embedding)]),
|
||||
updated_at: knex.raw('now()'),
|
||||
})))
|
||||
.onConflict(['media_id', 'face_index'])
|
||||
.ignore();
|
||||
|
||||
const processed = avatarIndex + 1;
|
||||
const [elapsedSec, elapsedNs] = process.hrtime(startTime);
|
||||
const elapsedMs = (elapsedSec * nsPerSec + elapsedNs) / 1e6;
|
||||
const avgMsPerEntry = elapsedMs / processed;
|
||||
const remainingMs = avgMsPerEntry * (avatarEntries.length - processed);
|
||||
const remainingMinutes = (remainingMs / 1000 / 60).toFixed(1);
|
||||
|
||||
logger.info(`Set biometrics for ${avatarEntry.media_id} (${processed}/${avatarEntries.length}, ~${remainingMinutes}m remaining)`);
|
||||
}, Promise.resolve());
|
||||
|
||||
const diffTime = process.hrtime(startTime);
|
||||
const totalMs = (diffTime[0] * nsPerSec + diffTime[1]) / 1e6;
|
||||
|
||||
logger.info(`Biometrics done in ${(totalMs / 1000).toFixed(1)}s for ${actorIds?.length || 'all'} actors and ${avatarEntries.length} avatars`);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
setBiometrics,
|
||||
};
|
||||
51
src/media.js
51
src/media.js
@@ -28,7 +28,6 @@ const chunk = require('./utils/chunk');
|
||||
const { get } = require('./utils/qu');
|
||||
const { fetchEntityReleaseIds } = require('./entity-releases');
|
||||
|
||||
// const pipeline = util.promisify(stream.pipeline);
|
||||
const streamQueue = taskQueue();
|
||||
|
||||
const s3 = new S3Client({
|
||||
@@ -516,16 +515,6 @@ async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, fil
|
||||
writeLazy(image, lazypath, info),
|
||||
]);
|
||||
|
||||
/*
|
||||
if (isProcessed) {
|
||||
// file already stored, remove temporary file
|
||||
await fsPromises.unlink(media.file.path);
|
||||
} else {
|
||||
// image not processed, simply move temporary file to final location
|
||||
await fsPromises.rename(media.file.path, path.join(config.media.path, filepath));
|
||||
}
|
||||
*/
|
||||
|
||||
await fsPromises.unlink(media.file.path);
|
||||
|
||||
if (config.s3.enabled) {
|
||||
@@ -731,31 +720,31 @@ async function fetchSource(source, baseMedia) {
|
||||
}
|
||||
|
||||
async function attempt(attempts = 1) {
|
||||
const hasher = new blake2.Hash('blake2b', { digestLength: 24 });
|
||||
let hasherReady = true;
|
||||
hasher.setEncoding('hex');
|
||||
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
|
||||
let tempFileTarget;
|
||||
|
||||
const hasher = blake2.createHash('blake2b', { digestLength: 24 });
|
||||
|
||||
let size = 0;
|
||||
|
||||
const hashStream = new stream.Transform({
|
||||
transform(streamChunk, _encoding, callback) {
|
||||
size += streamChunk.length;
|
||||
hasher.update(streamChunk);
|
||||
this.push(streamChunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}`);
|
||||
const tempFileTarget = fs.createWriteStream(tempFilePath);
|
||||
const hashStream = new stream.PassThrough();
|
||||
let size = 0;
|
||||
|
||||
hashStream.on('data', (streamChunk) => {
|
||||
size += streamChunk.length;
|
||||
|
||||
if (hasherReady) {
|
||||
hasher.write(streamChunk);
|
||||
}
|
||||
});
|
||||
tempFileTarget = fs.createWriteStream(tempFilePath);
|
||||
|
||||
const { mimetype } = source.stream
|
||||
? await streamQueue.push('fetchStreamSource', { source, tempFileTarget, hashStream })
|
||||
: await fetchHttpSource(source, tempFileTarget, hashStream);
|
||||
|
||||
hasher.end();
|
||||
const hash = hasher.digest('hex');
|
||||
|
||||
const hash = hasher.read();
|
||||
const [type, subtype] = mimetype.split('/');
|
||||
const extension = mime.getExtension(mimetype);
|
||||
|
||||
@@ -778,8 +767,10 @@ async function fetchSource(source, baseMedia) {
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
hasherReady = false;
|
||||
hasher.end();
|
||||
// hasherReady = false;
|
||||
// hasher.end();
|
||||
hashStream.destroy();
|
||||
tempFileTarget?.destroy();
|
||||
|
||||
if (error.code !== 'VERIFY_TYPE') {
|
||||
logger.warn(`Failed attempt ${attempts}/${maxAttempts} to fetch ${source.src}: ${error.message}`);
|
||||
|
||||
@@ -4,19 +4,28 @@ const unprint = require('unprint');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function extractEntryId(poster) {
|
||||
function extractEntryId(posters) {
|
||||
try {
|
||||
return slugify(new URL(poster).pathname.match(/\/images\/(.*?)\.[a-z]{3,4}/i)?.[1]?.replace(/smak.*/i, ''), '');
|
||||
const poster = [].concat(posters).filter(Boolean)[0];
|
||||
return slugify(new URL(poster.src || poster).pathname.match(/\/images\/(.*?)\.[a-z]{3,4}/i)?.[1]?.replace(/smak.*/i, ''), '');
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractTags(title) {
|
||||
if (!title) {
|
||||
function extractTags(title, titleComment) {
|
||||
if (!title && !titleComment) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (titleComment?.includes('<i>')) {
|
||||
const tagsMatch = titleComment.match(/<i>(.*?)<\/i>/)?.[1];
|
||||
|
||||
if (tagsMatch) {
|
||||
return tagsMatch.split('-').map((tag) => tag.trim().toLowerCase());
|
||||
}
|
||||
}
|
||||
|
||||
const firstTagIndex = title.match(/[A-Z]{2}/)?.index;
|
||||
|
||||
if (firstTagIndex) {
|
||||
@@ -40,14 +49,26 @@ function getPhotos(poster) {
|
||||
|
||||
if (photo === poster) {
|
||||
return {
|
||||
poster,
|
||||
poster: {
|
||||
src: poster,
|
||||
verifyType: 'image',
|
||||
},
|
||||
photos: [],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
poster: [photo, poster],
|
||||
photos: [poster],
|
||||
poster: [{
|
||||
src: photo,
|
||||
verifyType: 'image',
|
||||
}, {
|
||||
src: poster,
|
||||
verifyType: 'image',
|
||||
}],
|
||||
photos: [{
|
||||
src: poster,
|
||||
verifyType: 'image',
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -62,7 +83,7 @@ function scrapeAll(scenes, channel, parameters) {
|
||||
release.forceDeep = true;
|
||||
|
||||
release.title = query.content('a h5, .product-content p, .video_text');
|
||||
release.tags = extractTags(release.title);
|
||||
release.tags = extractTags(release.title, query.content('//a/comment()'));
|
||||
|
||||
const { poster, photos } = getPhotos(query.img('img[src*="/videos/images"], img[src*="/uploads/images"]'));
|
||||
|
||||
@@ -133,7 +154,8 @@ async function fetchLatestKanal(channel, page = 1, { parameters }) {
|
||||
|
||||
const res = await unprint.post(`${channel.origin}/pagination`, {
|
||||
k: page,
|
||||
status: 1,
|
||||
hidden_page_no: page - 1,
|
||||
status: true,
|
||||
}, {
|
||||
selectAll: '.video_bx',
|
||||
form: true,
|
||||
|
||||
31
src/tools/biometrics.js
Normal file
31
src/tools/biometrics.js
Normal file
@@ -0,0 +1,31 @@
|
||||
'use strict';
|
||||
|
||||
const yargs = require('yargs');
|
||||
const { hideBin } = require('yargs/helpers');
|
||||
|
||||
const knex = require('../knex');
|
||||
const { setBiometrics } = require('../biometrics');
|
||||
|
||||
const argv = yargs(hideBin(process.argv))
|
||||
.option('actor-ids', {
|
||||
alias: 'actors',
|
||||
type: 'array',
|
||||
describe: 'List of actor IDs to derive biometrics for',
|
||||
})
|
||||
.option('update', {
|
||||
alias: 'force',
|
||||
type: 'boolean',
|
||||
describe: 'Re-compute biometrics when already present',
|
||||
})
|
||||
.option('photos', {
|
||||
type: 'boolean',
|
||||
describe: 'Include secondary actor photos',
|
||||
})
|
||||
.argv;
|
||||
|
||||
async function init() {
|
||||
await setBiometrics(argv.actorIds, argv.update, argv.photos);
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,109 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
const args = require('yargs').argv;
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
const searchApi = new manticore.SearchApi(mantiClient);
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
const update = args.update;
|
||||
|
||||
async function fetchActors() {
|
||||
// manually select date of birth, otherwise it is retrieved in local timezone but interpreted as UTC...
|
||||
const actors = await knex.raw(`
|
||||
SELECT
|
||||
actors.*,
|
||||
actors_meta.*,
|
||||
date_of_birth AT TIME ZONE 'Europe/Amsterdam' AT TIME ZONE 'UTC' as dob
|
||||
FROM actors
|
||||
LEFT JOIN actors_meta ON actors_meta.actor_id = actors.id
|
||||
`);
|
||||
|
||||
return actors.rows;
|
||||
}
|
||||
|
||||
async function init() {
|
||||
if (update) {
|
||||
await utilsApi.sql('drop table if exists actors');
|
||||
await utilsApi.sql(`create table actors(
|
||||
id int,
|
||||
name text,
|
||||
slug string,
|
||||
entity_id int,
|
||||
gender string,
|
||||
date_of_birth timestamp,
|
||||
country string,
|
||||
has_avatar bool,
|
||||
mass int,
|
||||
height int,
|
||||
cup string,
|
||||
natural_boobs int,
|
||||
penis_length int,
|
||||
penis_girth int,
|
||||
stashed int,
|
||||
scenes int
|
||||
) min_prefix_len = '3'`);
|
||||
|
||||
const actors = await fetchActors();
|
||||
|
||||
const docs = actors.map((actor) => ({
|
||||
insert: {
|
||||
index: 'actors',
|
||||
id: actor.id,
|
||||
doc: {
|
||||
entity_id: actor.entity_id,
|
||||
name: actor.name,
|
||||
slug: actor.slug,
|
||||
gender: actor.gender || undefined,
|
||||
date_of_birth: actor.dob ? Math.round(actor.dob.getTime() / 1000) : undefined,
|
||||
has_avatar: !!actor.avatar_media_id,
|
||||
country: actor.birth_country_alpha2 || undefined,
|
||||
height: actor.height || undefined,
|
||||
mass: actor.weight || undefined, // weight is a reserved keyword in manticore
|
||||
cup: actor.cup || undefined,
|
||||
natural_boobs: actor.natural_boobs === null ? 0 : Number(actor.natural_boobs) + 1, // manticore bool does not seem to support null, and we need three states for natural_boobs: yes, no and unknown
|
||||
penis_length: actor.penis_length || undefined,
|
||||
penis_girth: actor.penis_girth || undefined,
|
||||
stashed: actor.stashed || 0,
|
||||
scenes: actor.scenes || 0,
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')).catch((error) => {
|
||||
console.log(error);
|
||||
});
|
||||
|
||||
console.log('data', data);
|
||||
knex.destroy();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await searchApi.search({
|
||||
index: 'actors',
|
||||
query: {
|
||||
equals: {
|
||||
has_avatar: 1,
|
||||
},
|
||||
},
|
||||
limit: 3,
|
||||
sort: [{ slug: 'asc' }],
|
||||
});
|
||||
|
||||
console.log(result);
|
||||
console.log(result.hits?.hits);
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,165 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
const args = require('yargs').argv;
|
||||
const { format } = require('date-fns');
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
// const searchApi = new manticore.SearchApi(mantiClient);
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
const update = args.update;
|
||||
|
||||
async function fetchMovies() {
|
||||
const movies = await knex.raw(`
|
||||
SELECT
|
||||
movies.id AS id,
|
||||
movies.title,
|
||||
movies.created_at,
|
||||
movies.date,
|
||||
movies_meta.stashed,
|
||||
entities.id as channel_id,
|
||||
entities.slug as channel_slug,
|
||||
entities.name as channel_name,
|
||||
parents.id as network_id,
|
||||
parents.slug as network_slug,
|
||||
parents.name as network_name,
|
||||
movies_covers IS NOT NULL as has_cover,
|
||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
||||
COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
|
||||
row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
|
||||
FROM movies
|
||||
LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
|
||||
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
|
||||
LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
|
||||
LEFT JOIN entities ON movies.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
|
||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
|
||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
|
||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
||||
LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
|
||||
LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
|
||||
LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
|
||||
GROUP BY
|
||||
movies.id,
|
||||
movies.title,
|
||||
movies.created_at,
|
||||
movies.date,
|
||||
movies_meta.stashed,
|
||||
movies_meta.stashed_scenes,
|
||||
movies_meta.stashed_total,
|
||||
entities.id,
|
||||
entities.name,
|
||||
entities.slug,
|
||||
entities.alias,
|
||||
parents.id,
|
||||
parents.name,
|
||||
parents.slug,
|
||||
parents.alias,
|
||||
movies_covers.*
|
||||
`);
|
||||
|
||||
return movies.rows;
|
||||
}
|
||||
|
||||
async function init() {
|
||||
if (update) {
|
||||
await utilsApi.sql('drop table if exists movies');
|
||||
await utilsApi.sql(`create table movies (
|
||||
id int,
|
||||
title text,
|
||||
title_filtered text,
|
||||
channel_id int,
|
||||
channel_name text,
|
||||
channel_slug text,
|
||||
network_id int,
|
||||
network_name text,
|
||||
network_slug text,
|
||||
entity_ids multi,
|
||||
actor_ids multi,
|
||||
actors text,
|
||||
tag_ids multi,
|
||||
tags text,
|
||||
meta text,
|
||||
date timestamp,
|
||||
has_cover bool,
|
||||
created_at timestamp,
|
||||
effective_date timestamp,
|
||||
stashed int,
|
||||
stashed_scenes int,
|
||||
stashed_total int,
|
||||
dupe_index int
|
||||
)`);
|
||||
|
||||
const movies = await fetchMovies();
|
||||
|
||||
console.log(movies.toSorted((movieA, movieB) => movieA.dupe_index - movieB.dupe_index));
|
||||
|
||||
const docs = movies.map((movie) => {
|
||||
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
|
||||
id: tag.f1,
|
||||
name: tag.f2,
|
||||
priority: tag.f3,
|
||||
alias: tag.f4,
|
||||
}])));
|
||||
|
||||
const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
||||
const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
||||
const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
|
||||
|
||||
return {
|
||||
replace: {
|
||||
index: 'movies',
|
||||
id: movie.id,
|
||||
doc: {
|
||||
title: movie.title || undefined,
|
||||
title_filtered: filteredTitle || undefined,
|
||||
date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
|
||||
created_at: Math.round(movie.created_at.getTime() / 1000),
|
||||
effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
|
||||
channel_id: movie.channel_id,
|
||||
channel_slug: movie.channel_slug,
|
||||
channel_name: movie.channel_name,
|
||||
network_id: movie.network_id || undefined,
|
||||
network_slug: movie.network_slug || undefined,
|
||||
network_name: movie.network_name || undefined,
|
||||
entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
|
||||
actor_ids: movie.actors.map((actor) => actor.f1),
|
||||
actors: movie.actors.map((actor) => actor.f2).join(),
|
||||
tag_ids: combinedTags.map((tag) => tag.id),
|
||||
tags: flatTags.join(' '),
|
||||
has_cover: movie.has_cover,
|
||||
meta: movie.date ? format(movie.date, 'y yy M MM MMM MMMM d dd') : undefined,
|
||||
stashed: movie.stashed || 0,
|
||||
stashed_scenes: movie.stashed_scenes || 0,
|
||||
stashed_total: movie.stashed_total || 0,
|
||||
dupe_index: movie.dupe_index || 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
console.log(docs.map((doc) => doc.replace));
|
||||
|
||||
const data = await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
|
||||
console.log('data', data);
|
||||
}
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,228 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
const args = require('yargs').argv;
|
||||
const { format } = require('date-fns');
|
||||
|
||||
const knex = require('../knex');
|
||||
const chunk = require('../utils/chunk');
|
||||
const filterTitle = require('../utils/filter-title');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
const update = args.update;
|
||||
|
||||
async function fetchScenes() {
|
||||
const scenes = await knex.raw(`
|
||||
SELECT
|
||||
releases.id AS id,
|
||||
releases.title,
|
||||
releases.created_at,
|
||||
releases.date,
|
||||
releases.entry_id,
|
||||
releases.shoot_id,
|
||||
scenes_meta.stashed,
|
||||
entities.id as channel_id,
|
||||
entities.slug as channel_slug,
|
||||
entities.name as channel_name,
|
||||
entities.alias as channel_aliases,
|
||||
parents.id as network_id,
|
||||
parents.slug as network_slug,
|
||||
parents.name as network_name,
|
||||
parents.alias as network_aliases,
|
||||
studios.id as studio_id,
|
||||
studios.slug as studio_slug,
|
||||
studios.name as studio_name,
|
||||
grandparents.id as parent_network_id,
|
||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name, local_tags.actor_id)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
||||
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
|
||||
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
|
||||
COALESCE(JSON_AGG(DISTINCT (releases_fingerprints.hash)) FILTER (WHERE releases_fingerprints.hash IS NOT NULL), '[]') as fingerprints,
|
||||
studios.showcased IS NOT false
|
||||
AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
|
||||
AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
|
||||
AND (releases_summaries.batch_showcased IS NOT false)
|
||||
AS showcased,
|
||||
row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
|
||||
FROM releases
|
||||
LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
|
||||
LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
|
||||
LEFT JOIN entities ON releases.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
|
||||
LEFT JOIN entities AS studios ON studios.id = releases.studio_id
|
||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
||||
LEFT JOIN releases_fingerprints ON releases_fingerprints.scene_id = releases.id
|
||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
||||
LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
|
||||
LEFT JOIN movies ON movies.id = movies_scenes.movie_id
|
||||
LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
|
||||
LEFT JOIN series ON series.id = series_scenes.serie_id
|
||||
GROUP BY
|
||||
releases.id,
|
||||
releases.title,
|
||||
releases.created_at,
|
||||
releases.date,
|
||||
releases.entry_id,
|
||||
releases.shoot_id,
|
||||
scenes_meta.stashed,
|
||||
releases_summaries.batch_showcased,
|
||||
entities.id,
|
||||
entities.name,
|
||||
entities.slug,
|
||||
entities.alias,
|
||||
parents.id,
|
||||
parents.name,
|
||||
parents.slug,
|
||||
parents.alias,
|
||||
grandparents.id,
|
||||
studios.id,
|
||||
studios.name,
|
||||
studios.slug,
|
||||
entities.showcased,
|
||||
parents.showcased,
|
||||
studios.showcased;
|
||||
`);
|
||||
|
||||
return scenes.rows;
|
||||
}
|
||||
|
||||
async function init() {
|
||||
if (update) {
|
||||
await utilsApi.sql('drop table if exists scenes');
|
||||
await utilsApi.sql(`create table scenes (
|
||||
id int,
|
||||
title text,
|
||||
title_filtered text,
|
||||
entry_id text,
|
||||
shoot_id text,
|
||||
channel_id int,
|
||||
channel_name text,
|
||||
channel_slug text,
|
||||
network_id int,
|
||||
network_name text,
|
||||
network_slug text,
|
||||
studio_id int,
|
||||
studio_name text,
|
||||
studio_slug text,
|
||||
entity_ids multi,
|
||||
actor_ids multi,
|
||||
actors text,
|
||||
tag_ids multi,
|
||||
tags text,
|
||||
movie_ids multi,
|
||||
movies text,
|
||||
serie_ids multi,
|
||||
series text,
|
||||
meta text,
|
||||
date timestamp,
|
||||
fingerprints text,
|
||||
is_showcased bool,
|
||||
created_at timestamp,
|
||||
effective_date timestamp,
|
||||
stashed int,
|
||||
dupe_index int
|
||||
)`);
|
||||
|
||||
await utilsApi.sql('drop table if exists scenes_tags');
|
||||
await utilsApi.sql(`create table scenes_tags (
|
||||
id int,
|
||||
scene_id int,
|
||||
tag_id int,
|
||||
actor_id int
|
||||
)`);
|
||||
|
||||
console.log('Recreated scenes table');
|
||||
console.log('Fetching scenes from primary database');
|
||||
|
||||
const scenes = await fetchScenes();
|
||||
|
||||
console.log('Fetched scenes from primary database');
|
||||
|
||||
const docs = scenes.flatMap((scene) => {
|
||||
const flatActors = scene.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
||||
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => (tag.f4 ? `${tag.f2} ${tag.f4}` : tag.f2).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
||||
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
|
||||
|
||||
return [
|
||||
{
|
||||
replace: {
|
||||
index: 'scenes',
|
||||
id: scene.id,
|
||||
doc: {
|
||||
title: scene.title || undefined,
|
||||
title_filtered: filteredTitle || undefined,
|
||||
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
||||
created_at: Math.round(scene.created_at.getTime() / 1000),
|
||||
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
||||
is_showcased: scene.showcased,
|
||||
entry_id: scene.entry_id || undefined,
|
||||
shoot_id: scene.shoot_id || undefined,
|
||||
channel_id: scene.channel_id,
|
||||
channel_slug: scene.channel_slug,
|
||||
channel_name: [].concat(scene.channel_name, scene.channel_aliases).join(' '),
|
||||
network_id: scene.network_id || undefined,
|
||||
network_slug: scene.network_slug || undefined,
|
||||
network_name: [].concat(scene.network_name, scene.network_aliases).join(' ') || undefined,
|
||||
studio_id: scene.studio_id || undefined,
|
||||
studio_slug: scene.studio_slug || undefined,
|
||||
studio_name: scene.studio_name || undefined,
|
||||
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
|
||||
actor_ids: scene.actors.map((actor) => actor.f1),
|
||||
actors: scene.actors.map((actor) => actor.f2).join(),
|
||||
tag_ids: scene.tags.map((tag) => tag.f1),
|
||||
tags: flatTags.join(' '),
|
||||
movie_ids: scene.movies.map((movie) => movie.f1),
|
||||
movies: scene.movies.map((movie) => movie.f2).join(' '),
|
||||
serie_ids: scene.series.map((serie) => serie.f1),
|
||||
series: scene.series.map((serie) => serie.f2).join(' '),
|
||||
fingerprints: scene.fingerprints.join(' '),
|
||||
meta: scene.date ? format(scene.date, 'y yy M MM MMM MMMM d dd') : undefined,
|
||||
stashed: scene.stashed || 0,
|
||||
dupe_index: scene.dupe_index || 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
...scene.tags.map((tag) => ({
|
||||
replace: {
|
||||
index: 'scenes_tags',
|
||||
// id: scene.id,
|
||||
doc: {
|
||||
scene_id: scene.id,
|
||||
tag_id: tag.f1,
|
||||
actor_id: tag.f5,
|
||||
},
|
||||
},
|
||||
})),
|
||||
];
|
||||
});
|
||||
|
||||
// const accData = chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
|
||||
chunk(docs, 10000).reduce(async (chain, docsChunk, index, array) => {
|
||||
const acc = await chain;
|
||||
const data = await indexApi.bulk(docsChunk.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
|
||||
console.log(`Seeded ${index + 1}/${array.length}, errors: ${data.errors} ${data.error}`);
|
||||
|
||||
return acc.concat(data.items);
|
||||
}, Promise.resolve([]));
|
||||
|
||||
// console.log('data', accData);
|
||||
}
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,88 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const manticore = require('manticoresearch');
|
||||
|
||||
const knex = require('../knex');
|
||||
const chunk = require('../utils/chunk');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
|
||||
mantiClient.basePath = `http://${config.database.manticore.host}:${config.database.manticore.httpPort}`;
|
||||
|
||||
const utilsApi = new manticore.UtilsApi(mantiClient);
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
async function syncStashes(domain = 'scene') {
|
||||
await utilsApi.sql(`truncate table ${domain}s_stashed`);
|
||||
|
||||
const stashes = await knex(`stashes_${domain}s`)
|
||||
.select(
|
||||
`stashes_${domain}s.id as stashed_id`,
|
||||
`stashes_${domain}s.${domain}_id`,
|
||||
'stashes.id as stash_id',
|
||||
'stashes.user_id as user_id',
|
||||
`stashes_${domain}s.created_at as created_at`,
|
||||
)
|
||||
.leftJoin('stashes', 'stashes.id', `stashes_${domain}s.stash_id`);
|
||||
|
||||
await chunk(stashes, 1000).reduce(async (chain, stashChunk, index) => {
|
||||
await chain;
|
||||
|
||||
const stashDocs = stashChunk.map((stash) => ({
|
||||
replace: {
|
||||
index: `${domain}s_stashed`,
|
||||
id: stash.stashed_id,
|
||||
doc: {
|
||||
[`${domain}_id`]: stash[`${domain}_id`],
|
||||
stash_id: stash.stash_id,
|
||||
user_id: stash.user_id,
|
||||
created_at: Math.round(stash.created_at.getTime() / 1000),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
|
||||
console.log(`Synced ${index * 1000 + stashChunk.length}/${stashes.length} ${domain} stashes`);
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
async function init() {
|
||||
await utilsApi.sql('drop table if exists scenes_stashed');
|
||||
|
||||
await utilsApi.sql(`create table if not exists scenes_stashed (
|
||||
scene_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql('drop table if exists movies_stashed');
|
||||
|
||||
await utilsApi.sql(`create table if not exists movies_stashed (
|
||||
movie_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await utilsApi.sql('drop table if exists actors_stashed');
|
||||
|
||||
await utilsApi.sql(`create table if not exists actors_stashed (
|
||||
actor_id int,
|
||||
stash_id int,
|
||||
user_id int,
|
||||
created_at timestamp
|
||||
)`);
|
||||
|
||||
await syncStashes('scene');
|
||||
await syncStashes('actor');
|
||||
await syncStashes('movie');
|
||||
|
||||
console.log('Done!');
|
||||
|
||||
knex.destroy();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,12 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const { updateSceneSearch, updateMovieSearch } = require('../update-search');
|
||||
|
||||
async function init() {
|
||||
await updateSceneSearch();
|
||||
await updateMovieSearch();
|
||||
|
||||
process.exit();
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,436 +1,48 @@
|
||||
'use strict';
|
||||
|
||||
const manticore = require('manticoresearch');
|
||||
const { format } = require('date-fns');
|
||||
const config = require('config');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const knex = require('./knex');
|
||||
const logger = require('./logger')(__filename);
|
||||
const bulkInsert = require('./utils/bulk-insert');
|
||||
const chunk = require('./utils/chunk');
|
||||
const filterTitle = require('./utils/filter-title');
|
||||
|
||||
const mantiClient = new manticore.ApiClient();
|
||||
const indexApi = new manticore.IndexApi(mantiClient);
|
||||
|
||||
async function updateManticoreStashedScenes(docs) {
|
||||
await chunk(docs, 1000).reduce(async (chain, docsChunk) => {
|
||||
await chain;
|
||||
|
||||
const sceneIds = docsChunk.filter((doc) => !!doc.replace).map((doc) => doc.replace.id);
|
||||
|
||||
const stashes = await knex('stashes_scenes')
|
||||
.select('stashes_scenes.id as stashed_id', 'stashes_scenes.scene_id', 'stashes_scenes.created_at', 'stashes.id as stash_id', 'stashes.user_id as user_id')
|
||||
.leftJoin('stashes', 'stashes.id', 'stashes_scenes.stash_id')
|
||||
.whereIn('scene_id', sceneIds);
|
||||
|
||||
const stashDocs = docsChunk.filter((doc) => doc.replace).flatMap((doc) => {
|
||||
const sceneStashes = stashes.filter((stash) => stash.scene_id === doc.replace.id);
|
||||
|
||||
if (sceneStashes.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const stashDoc = sceneStashes.map((stash) => ({
|
||||
replace: {
|
||||
index: 'scenes_stashed',
|
||||
id: stash.stashed_id,
|
||||
doc: {
|
||||
// ...doc.replace.doc,
|
||||
scene_id: doc.replace.id,
|
||||
user_id: stash.user_id,
|
||||
stash_id: stash.stash_id,
|
||||
created_at: Math.round(stash.created_at.getTime() / 1000),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
return stashDoc;
|
||||
});
|
||||
|
||||
if (stashDocs.length > 0) {
|
||||
await indexApi.bulk(stashDocs.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
}
|
||||
|
||||
const deleteSceneIds = docs.filter((doc) => doc.delete).map((doc) => doc.delete.id);
|
||||
|
||||
if (deleteSceneIds.length > 0) {
|
||||
await indexApi.callDelete({
|
||||
index: 'scenes_stashed',
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
in: {
|
||||
scene_id: deleteSceneIds,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
async function updateManticoreSceneSearch(releaseIds) {
|
||||
logger.info(`Updating Manticore search documents for ${releaseIds ? releaseIds.length : 'all' } scenes`);
|
||||
|
||||
const scenes = await knex.raw(`
|
||||
SELECT
|
||||
releases.id AS id,
|
||||
releases.title,
|
||||
releases.created_at,
|
||||
releases.date,
|
||||
releases.shoot_id,
|
||||
scenes_meta.stashed,
|
||||
entities.id as channel_id,
|
||||
entities.slug as channel_slug,
|
||||
entities.name as channel_name,
|
||||
parents.id as network_id,
|
||||
parents.slug as network_slug,
|
||||
parents.name as network_name,
|
||||
studios.id as studio_id,
|
||||
studios.slug as studio_slug,
|
||||
studios.name as studio_name,
|
||||
grandparents.id as parent_network_id,
|
||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
||||
COALESCE(JSON_AGG(DISTINCT (movies.id, movies.title)) FILTER (WHERE movies.id IS NOT NULL), '[]') as movies,
|
||||
COALESCE(JSON_AGG(DISTINCT (series.id, series.title)) FILTER (WHERE series.id IS NOT NULL), '[]') as series,
|
||||
studios.showcased IS NOT false
|
||||
AND (entities.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
|
||||
AND (parents.showcased IS NOT false OR COALESCE(entities.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
|
||||
AND (releases_summaries.batch_showcased IS NOT false)
|
||||
AS showcased,
|
||||
row_number() OVER (PARTITION BY releases.entry_id, parents.id ORDER BY releases.effective_date DESC) as dupe_index
|
||||
FROM releases
|
||||
LEFT JOIN releases_summaries ON releases_summaries.release_id = releases.id
|
||||
LEFT JOIN scenes_meta ON scenes_meta.scene_id = releases.id
|
||||
LEFT JOIN entities ON releases.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN entities AS grandparents ON grandparents.id = parents.parent_id
|
||||
LEFT JOIN entities AS studios ON studios.id = releases.studio_id
|
||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
||||
LEFT JOIN movies_scenes ON movies_scenes.scene_id = releases.id
|
||||
LEFT JOIN movies ON movies.id = movies_scenes.movie_id
|
||||
LEFT JOIN series_scenes ON series_scenes.scene_id = releases.id
|
||||
LEFT JOIN series ON series.id = series_scenes.serie_id
|
||||
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
|
||||
GROUP BY
|
||||
releases.id,
|
||||
releases.title,
|
||||
releases.created_at,
|
||||
releases.date,
|
||||
releases.shoot_id,
|
||||
scenes_meta.stashed,
|
||||
releases_summaries.batch_showcased,
|
||||
entities.id,
|
||||
entities.name,
|
||||
entities.slug,
|
||||
entities.alias,
|
||||
entities.showcased,
|
||||
parents.id,
|
||||
parents.name,
|
||||
parents.slug,
|
||||
parents.alias,
|
||||
grandparents.id,
|
||||
studios.id,
|
||||
studios.name,
|
||||
studios.slug,
|
||||
parents.showcased,
|
||||
studios.showcased
|
||||
`, releaseIds && [releaseIds]);
|
||||
|
||||
const scenesById = Object.fromEntries(scenes.rows.map((scene) => [scene.id, scene]));
|
||||
|
||||
const docs = releaseIds.map((sceneId) => {
|
||||
const scene = scenesById[sceneId];
|
||||
|
||||
if (!scene) {
|
||||
return {
|
||||
delete: {
|
||||
index: 'scenes',
|
||||
id: sceneId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const flatActors = scene.actors.flatMap((actor) => actor.f2.split(' '));
|
||||
const flatTags = scene.tags.filter((tag) => tag.f3 > 6).flatMap((tag) => [tag.f2].concat(tag.f4)).filter(Boolean); // only make top tags searchable to minimize cluttered results
|
||||
const filteredTitle = filterTitle(scene.title, [...flatActors, ...flatTags]);
|
||||
|
||||
return {
|
||||
replace: {
|
||||
index: 'scenes',
|
||||
id: scene.id,
|
||||
doc: {
|
||||
title: scene.title || undefined,
|
||||
title_filtered: filteredTitle || undefined,
|
||||
date: scene.date ? Math.round(scene.date.getTime() / 1000) : undefined,
|
||||
created_at: Math.round(scene.created_at.getTime() / 1000),
|
||||
effective_date: Math.round((scene.date || scene.created_at).getTime() / 1000),
|
||||
is_showcased: scene.showcased,
|
||||
shoot_id: scene.shoot_id || undefined,
|
||||
channel_id: scene.channel_id,
|
||||
channel_slug: scene.channel_slug,
|
||||
channel_name: scene.channel_name,
|
||||
network_id: scene.network_id || undefined,
|
||||
network_slug: scene.network_slug || undefined,
|
||||
network_name: scene.network_name || undefined,
|
||||
studio_id: scene.studio_id || undefined,
|
||||
studio_slug: scene.studio_slug || undefined,
|
||||
studio_name: scene.studio_name || undefined,
|
||||
entity_ids: [scene.channel_id, scene.network_id, scene.parent_network_id, scene.studio_id].filter(Boolean), // manticore does not support OR, this allows IN
|
||||
actor_ids: scene.actors.map((actor) => actor.f1),
|
||||
actors: scene.actors.map((actor) => actor.f2).join(),
|
||||
tag_ids: scene.tags.map((tag) => tag.f1),
|
||||
tags: flatTags.join(' '), // only make top tags searchable to minimize cluttered results
|
||||
movie_ids: scene.movies.map((movie) => movie.f1),
|
||||
movies: scene.movies.map((movie) => movie.f2).join(' '),
|
||||
serie_ids: scene.series.map((serie) => serie.f1),
|
||||
series: scene.series.map((serie) => serie.f2).join(' '),
|
||||
meta: scene.date ? format(scene.date, 'y yy M MMM MMMM d') : undefined,
|
||||
stashed: scene.stashed || 0,
|
||||
dupe_index: scene.dupe_index || 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
if (docs.length === 0) {
|
||||
async function syncWeb(domain, ids) {
|
||||
if (!ids || ids.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n')),
|
||||
updateManticoreStashedScenes(docs),
|
||||
]);
|
||||
}
|
||||
await knex('sync').insert({ domain, item_ids: ids });
|
||||
|
||||
async function updateSqlSceneSearch(releaseIds) {
|
||||
logger.info(`Updating SQL search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
|
||||
|
||||
const documents = await knex.raw(`
|
||||
SELECT
|
||||
releases.id AS release_id,
|
||||
TO_TSVECTOR(
|
||||
'english',
|
||||
COALESCE(releases.title, '') || ' ' ||
|
||||
releases.entry_id || ' ' ||
|
||||
entities.name || ' ' ||
|
||||
entities.slug || ' ' ||
|
||||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
||||
COALESCE(parents.name, '') || ' ' ||
|
||||
COALESCE(parents.slug, '') || ' ' ||
|
||||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
||||
COALESCE(releases.shoot_id, '') || ' ' ||
|
||||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
|
||||
) as document
|
||||
FROM releases
|
||||
LEFT JOIN entities ON releases.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
|
||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
|
||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
|
||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
||||
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
|
||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
||||
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
|
||||
GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
||||
`, releaseIds && [releaseIds]);
|
||||
|
||||
if (documents.rows?.length > 0) {
|
||||
await bulkInsert('releases_search', documents.rows, ['release_id']);
|
||||
if (config.webApi.enabled) {
|
||||
await unprint.post(`${config.webApi.address}/sync`, null, {
|
||||
headers: {
|
||||
'api-user': config.webApi.apiUserId,
|
||||
'api-key': config.webApi.apiKey,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
|
||||
}
|
||||
|
||||
async function updateSceneSearch(releaseIds) {
|
||||
await knex.raw('REFRESH MATERIALIZED VIEW scenes_meta;');
|
||||
await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
|
||||
|
||||
await updateSqlSceneSearch(releaseIds);
|
||||
await updateManticoreSceneSearch(releaseIds);
|
||||
}
|
||||
|
||||
async function updateManticoreMovieSearch(movieIds) {
|
||||
logger.info(`Updating Manticore search documents for ${movieIds ? movieIds.length : 'all' } movies`);
|
||||
|
||||
const movies = await knex.raw(`
|
||||
SELECT
|
||||
movies.id AS id,
|
||||
movies.title,
|
||||
movies.created_at,
|
||||
movies.date,
|
||||
movies_meta.stashed,
|
||||
entities.id as channel_id,
|
||||
entities.slug as channel_slug,
|
||||
entities.name as channel_name,
|
||||
parents.id as network_id,
|
||||
parents.slug as network_slug,
|
||||
parents.name as network_name,
|
||||
movies_covers IS NOT NULL as has_cover,
|
||||
COALESCE(JSON_AGG(DISTINCT (actors.id, actors.name)) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(JSON_AGG(DISTINCT (tags.id, tags.name, tags.priority, tags_aliases.name)) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags,
|
||||
COALESCE(JSON_AGG(DISTINCT (movie_tags.id, movie_tags.name, movie_tags.priority, movie_tags_aliases.name)) FILTER (WHERE movie_tags.id IS NOT NULL), '[]') as movie_tags,
|
||||
row_number() OVER (PARTITION BY movies.entry_id, parents.id ORDER BY movies.effective_date DESC) as dupe_index
|
||||
FROM movies
|
||||
LEFT JOIN movies_meta ON movies_meta.movie_id = movies.id
|
||||
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
|
||||
LEFT JOIN movies_tags ON movies_tags.movie_id = movies.id
|
||||
LEFT JOIN entities ON movies.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = movies_scenes.scene_id
|
||||
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = movies_scenes.scene_id
|
||||
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = movies_scenes.scene_id
|
||||
LEFT JOIN actors ON local_actors.actor_id = actors.id
|
||||
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
|
||||
LEFT JOIN tags ON local_tags.tag_id = tags.id
|
||||
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
|
||||
LEFT JOIN tags as movie_tags ON movies_tags.tag_id = movie_tags.id
|
||||
LEFT JOIN tags as movie_tags_aliases ON movies_tags.tag_id = movie_tags_aliases.alias_for AND movie_tags_aliases.secondary = true
|
||||
LEFT JOIN movies_covers ON movies_covers.movie_id = movies.id
|
||||
${movieIds ? 'WHERE movies.id = ANY(?)' : ''}
|
||||
GROUP BY
|
||||
movies.id,
|
||||
movies.title,
|
||||
movies.created_at,
|
||||
movies.date,
|
||||
movies_meta.stashed,
|
||||
movies_meta.stashed_scenes,
|
||||
movies_meta.stashed_total,
|
||||
entities.id,
|
||||
entities.name,
|
||||
entities.slug,
|
||||
entities.alias,
|
||||
parents.id,
|
||||
parents.name,
|
||||
parents.slug,
|
||||
parents.alias,
|
||||
movies_covers.*
|
||||
`, movieIds && [movieIds]);
|
||||
|
||||
const moviesById = Object.fromEntries(movies.rows.map((movie) => [movie.id, movie]));
|
||||
|
||||
const docs = movieIds.map((movieId) => {
|
||||
const movie = moviesById[movieId];
|
||||
|
||||
if (!movie) {
|
||||
return {
|
||||
delete: {
|
||||
index: 'movies',
|
||||
id: movieId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const combinedTags = Object.values(Object.fromEntries(movie.tags.concat(movie.movie_tags).map((tag) => [tag.f1, {
|
||||
id: tag.f1,
|
||||
name: tag.f2,
|
||||
priority: tag.f3,
|
||||
alias: tag.f4,
|
||||
}])));
|
||||
|
||||
const flatActors = movie.actors.flatMap((actor) => actor.f2.match(/[\w']+/g)); // match word characters to filter out brackets etc.
|
||||
const flatTags = combinedTags.filter((tag) => tag.priority > 6).flatMap((tag) => (tag.alias ? `${tag.name} ${tag.alias}` : tag.name).match(/[\w']+/g)); // only make top tags searchable to minimize cluttered results
|
||||
const filteredTitle = movie.title && [...flatActors, ...flatTags].reduce((accTitle, tag) => accTitle.replace(new RegExp(tag.replace(/[^\w\s]+/g, ''), 'gi'), ''), movie.title).trim().replace(/\s{2,}/g, ' ');
|
||||
|
||||
return {
|
||||
replace: {
|
||||
index: 'movies',
|
||||
id: movie.id,
|
||||
doc: {
|
||||
title: movie.title || undefined,
|
||||
title_filtered: filteredTitle || undefined,
|
||||
date: movie.date ? Math.round(movie.date.getTime() / 1000) : undefined,
|
||||
created_at: Math.round(movie.created_at.getTime() / 1000),
|
||||
effective_date: Math.round((movie.date || movie.created_at).getTime() / 1000),
|
||||
channel_id: movie.channel_id,
|
||||
channel_slug: movie.channel_slug,
|
||||
channel_name: movie.channel_name,
|
||||
network_id: movie.network_id || undefined,
|
||||
network_slug: movie.network_slug || undefined,
|
||||
network_name: movie.network_name || undefined,
|
||||
entity_ids: [movie.channel_id, movie.network_id].filter(Boolean), // manticore does not support OR, this allows IN
|
||||
actor_ids: movie.actors.map((actor) => actor.f1),
|
||||
actors: movie.actors.map((actor) => actor.f2).join(),
|
||||
tag_ids: combinedTags.map((tag) => tag.id),
|
||||
tags: flatTags.join(' '),
|
||||
has_cover: movie.has_cover,
|
||||
meta: movie.date ? format(movie.date, 'y yy M MMM MMMM d') : undefined,
|
||||
stashed: movie.stashed || 0,
|
||||
stashed_scenes: movie.stashed_scenes || 0,
|
||||
stashed_total: movie.stashed_total || 0,
|
||||
dupe_index: movie.dupe_index || 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
if (docs.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await indexApi.bulk(docs.map((doc) => JSON.stringify(doc)).join('\n'));
|
||||
}
|
||||
|
||||
async function updateSqlMovieSearch(movieIds, target = 'movie') {
|
||||
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
|
||||
|
||||
const documents = await knex.raw(`
|
||||
SELECT
|
||||
${target}s.id AS ${target}_id,
|
||||
TO_TSVECTOR(
|
||||
'english',
|
||||
COALESCE(${target}s.title, '') || ' ' ||
|
||||
entities.name || ' ' ||
|
||||
entities.slug || ' ' ||
|
||||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
||||
COALESCE(parents.name, '') || ' ' ||
|
||||
COALESCE(parents.slug, '') || ' ' ||
|
||||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
||||
COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
|
||||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
||||
STRING_AGG(COALESCE(tags.name, ''), ' ')
|
||||
) as document
|
||||
FROM ${target}s
|
||||
LEFT JOIN entities ON ${target}s.entity_id = entities.id
|
||||
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
|
||||
LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
|
||||
LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
|
||||
LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
|
||||
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
|
||||
LEFT JOIN actors ON actors.id = releases_actors.actor_id
|
||||
LEFT JOIN tags ON tags.id = releases_tags.tag_id
|
||||
${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
|
||||
GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
|
||||
`, movieIds && [movieIds]);
|
||||
|
||||
if (documents.rows?.length > 0) {
|
||||
await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
|
||||
}
|
||||
await syncWeb('scene', releaseIds);
|
||||
}
|
||||
|
||||
async function updateMovieSearch(releaseIds) {
|
||||
await knex.raw('REFRESH MATERIALIZED VIEW movies_meta;');
|
||||
|
||||
await updateSqlMovieSearch(releaseIds);
|
||||
await updateManticoreMovieSearch(releaseIds);
|
||||
await syncWeb('movie', releaseIds);
|
||||
}
|
||||
|
||||
async function updateActorSearch(actorIds) {
|
||||
await knex.raw('REFRESH MATERIALIZED VIEW actors_meta;');
|
||||
|
||||
await syncWeb('actor', actorIds);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
updateSceneSearch,
|
||||
updateMovieSearch,
|
||||
updateActorSearch,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user