From 4b310e9dfa5ece67456ebcebf8b4d011da15ce27 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Thu, 19 Mar 2020 01:54:25 +0100 Subject: [PATCH] Added configurable proxy to HTTP module (also used by qu). Added network and site URL to search documents. --- config/default.js | 13 +++++ package-lock.json | 5 ++ package.json | 1 + public/img/logos/gamma/misc/powered.jpg | Bin 0 -> 12541 bytes src/app.js | 7 +++ src/releases.js | 4 +- src/utils/http.js | 67 ++++++++++++++---------- src/utils/qu.js | 6 +-- 8 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 public/img/logos/gamma/misc/powered.jpg diff --git a/config/default.js b/config/default.js index c96c49a6..7165b2b1 100644 --- a/config/default.js +++ b/config/default.js @@ -139,6 +139,19 @@ module.exports = { 'freeones', 'freeonesLegacy', ], + proxy: { + enable: false, + host: '', + port: 8888, + hostnames: [ + 'www.vixen.com', + 'www.blacked.com', + 'www.blackedraw.com', + 'www.tushy.com', + 'www.tushyraw.com', + 'www.deeper.com', + ], + }, fetchAfter: [1, 'week'], nullDateLimit: 3, media: { diff --git a/package-lock.json b/package-lock.json index 8829469f..206458f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11050,6 +11050,11 @@ } } }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index 775519d4..566d8834 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "template-format": "^1.2.5", "tough-cookie": "^3.0.1", "tty-table": "^2.8.12", + "tunnel": "0.0.6", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "vue": "^2.6.11", diff --git a/public/img/logos/gamma/misc/powered.jpg b/public/img/logos/gamma/misc/powered.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9c1380b3ddcf55431044e378856291aad5e377e5 GIT binary patch literal 12541 zcmch7cR-WL)^}*4D6WWrf`EboQWp@A&Mr+sh?LMmS*k(^kpu`GbXTQgsM3*6AjAM6 zkN}aDUWE`^Xi}w1@9?qrz3+YRx8J+>?%qGXndh0AoO5QL^UVC_%$#|0I&?Y-xT2@? zNC!Yi4*)owMZoDYfKAKW5#k4+126&r0G%_^D&X=DPA~^&fYZk_3f(CU@C$&E;kR&> zGM-~RyP2678JU>RpF78V?%a8n^B2y-c@~ySzlF1umG#mkw%@|Fe*`*4CZ_Yu=f7iN z`R>}~%a{K_|I>xjj{vswfOG~5105THo{f%yjqbD&Kt5BOj){(r?o83Y2L?L&bIj)% znJ%0W=^6en79BkU(>X@w^H%_L3=H%P49v{uSS~RAR_~0%#>jN_+--Jd=^yNkJbMLX zuAP7Jqj8)G2Nwz_7+-1jzQF-bF->sxzWy>tRufU-h3EWo=dQLe+nE9kbW98v&YivC z-&CMuqi0~eE&an)CL{ZEo-a`EaSgpP?BBi!{Ahe_-2pE6GA>7!qx{!9oXn>qfJ+Q# zTCy>)0n`C&lYguJCz4C*ML~~8>SR8Zf#GAWfrdrXCfq;dPTWq-_DBAe&=*U{>aIRcEZ5cUnZ zqF*A_ZywYfenB0(`2&I#w-LLa1P}VR07r<$`L)-&+r?k$_U5TQ6PIF_Og=NL_mpJS z-{2o8%x0d>GFK0jQO5u^zH3n{3*K`IvC0hPj|s9q{?>YeR|WiqW|>UNf^0xK1#7J( z>6H*;Jv4ep;4Yv8x4#yVUMf-D5;C*-ai)LR?J@P@h^G`Q@cpw+vj^5DSSRd-}lPt4NH`g*S z7ZpIF`@%AxsLT3?lPhd=#}v}iWCa#e+B=l329zN#=#8nTKO6aCyRFdc#-BL)8J@6e z$N_$rG56B~0;EZZDrFc0Y|YX=s+hI%Q3HMjkxrJ2b`wiqozYO&AR2tOc~9XHk<5Vc znIpxFJSxmD=_+Dv2WWECS!w0sB`N>6@-++%sa*~}t1{EFX{4AIF1Ilxm#k!xoS4=+ z2xKK7%whX$@&u>0N2(^Jpl`N+CdvYnh9!Av(*L@B3dp}f>u;ew9fm&NeHIRLNa|BS z?-)O_8aM%Iwz-g4DG!q$p}!@YP=!mD1dXC222V{@e`F%Py|sdtg)QUyE`Skz}`CFL8|S25US6PwGLFhsMZyf z>Ehur|4~c=n-AKafAO_8WpH0F8ew--Fj&jlr}0;z^zY^G+cj0$PIL9$z+}VmGBp>$ zHxfCsat)9yS)#xG@^Ml^>vcAFs2g3;x!+|6lJW8P$m<3W?y;JvA($AwRP>SZBK zq__j+Qf(UB-tI?Y27F$i#FE?3*ArXOT=~9=3cJy`-`jg_LcagsqwqibMh0f@Fa6wpJKlUcD62XTyx%UbCH;&2WFS% zPg2lLo|Y3O06sy^OIB5$29(kMxq=GEA}*d!15eqs2xX0`W0>T;4>%@bg?;yT24uj=HRlD0-t zf-opIrNMj9yLIMs4~s3vGsXSo+NMI`%ncPDP^FA&=}k2L+n3cFp4?I;T`zaWGBR1p z-peR^h5maK{#6MCx!Hk|Z?U%C0UP!RfAOnQ_%%!JbPgk~ZYHB`y?l{>SU#qp2N)4L z+i;r3-%+ewGTCMFjc|uNfmE-cp#*cprZmhOS?&7+FDXi#0w|XtI0?ueit>NKP`-TQ!r4dGM`j ziU5}HqR7+Ok3u3>Q`dH29WRGSatfZQ9^uo+s@TsrFLSXH);~;tFk7b$$+30Lt`C~- zjGhdN9^Wg~~d3Du>4CT*j$YrK2$`G(prxF!|jA7li-B$hcv*@|G` zMn=J!ssl~EkH(}V2nH4GJYQaUy>5`EG4osnT{Ryy3?plAl7RH(Nu9#C<7vn4HP2`(!pw6dWwO2&glA*BN)yupu3KE-eW&qPhyLHoD-KSp zy7r|b_g1jW1g(#gD_SuxJ~ojlc{F+3%qG~HRMwpog-#m3WwGahguYL@kt)pT1=2Wo zm=@fjO2S!)>)72EavQuM+BvB2$WFQ0R(4h73o5F=e6hcX`AsoN%yqJ=ezkSV)6;5{ zHa3~%v{SRX>ti>v=@KJ{NbW~e_qs7j`FK!W_QoHyu0N4);jf?wJJY*SI2K1fo*;tU z7W-#yRUW+9**Mhw$q}ByOGZKonDs06W?-LkaI%NcghZ~h6E+8JWQ*T04h>9?#u`e4 zU-m#;o?Fxg3+=jVqS9o=3Y;l$;}LOG34dqb7!ddr$niw2Qd!g9!s&UnN`BRNB&+u; zLUsLaxQF?3;^Gw5X?Um^7-zoz(b+!FL4 zY0O!yOFcr~W0*f8Z8lkYn0Y-!=)P^ETRo^Ss~y=3VL2WN-K|HQ0Vp)P_jIF9JtST9q@jnIN zwRdW_lw5lqql{vP^1!5(6-0(QcJcm5P5K7-+j9dcSNnaBsMcIVj;_uLgrd*J)@3?13rv676ATeg2@Bt(jIF`B-F>lb?vFdQ zeU{6*)yn4I@TWCG55;Z@Y2R%Ms5GMexYYdY)lpS$kmb9wXb3RR7q@+hI{@00R9$@5 z-nH;kz%{c8Ff~m-%_L~py}-~lCDW3|ablC~pTcw3sni{%%Fh#r2{1PSsc<~FN38lv z6l1q=6A}`NuM4nQ63)1w8KcIS0zoa<q5Yj1bnia}apFtu$(EgK4URa>ubYAHAcB&$fT$H~{Q$>p*SKN4LNcu_ffVdb?3rCfW#J$nwHcsf{#o6Kk>F)oE=hDdWEBp6g+-;}M%sY8n5A z^IKW!i^x@pu7oiKpWT`_I{Pb_TP>g39keEpc#p7NBV`afKHy`!gi;G>c^V(+>1g+^ zyz-FPqkK2PqfOf_%Az+qwXO@~CR&vZj$FFD(%2C0#OC<4zCq`$LwT~9G%$#j1bG-| z?n!AfDX+g3C_$=|O|w%#WPNJ8D1-DI)s!s=uh1zaS+%C}n@&%i`vFExFM)lNs%;*s zV18L0G;=-B6cQ0l>1=YGozo!p=}9qRR798y{kLrnOQ1r-FUR*kKk^@qZd$cH1*~06 z^l~?o5uxI=2BbwxhCxr{>MNJm=!z~c?A;GJ1td9zG_Qs{B`~ecaD{JAa{4A%TgV0ROb`N zNH5Ku%GnHB@%pa@hN#>z<_oqQ+eQ8PS3di~9&vY+Q5S_5?YNA`p_S8Cemg!2yhtOZ zX9R=SiW2WiDY3{FeC2p4s>~@Y*0I-W2kU@0j>vj@AQm1SZ}Agp6G?V*uQ3x|>_i{a zVz69dTLvo!F7=GP#F*5fK zSeg_<>sf{$M9UpKPM#*uRrv*<0#pL*+@{0=s_)>5(eX6Bh+%saTPIu!o|D`bGn&*z zXtZIkzx$>9u#_lShC#ScqIblT& zSkjZ6v*gmr6#{b}jVH!``l39cL$Pn6-pBboI|^`2Ral!;4b5n=bjFCjwUOhC zLRJn6^mf@@Gc{FvpFwKhww*}&Xn|dT(S@HePXum|R&yEMNS_{JX0Qk<*$Y5b>=8l&9c%npBX=&xOr9HB4fF(WX{ysEsrMM;3pdTmkiw> z2y-e~46mpb&zlOAXjdq|XUAgFHkanKjTPU=eLtacg(S=pAdVmlu=QL*6_W7m<+1xH z0l$Izm$l2Nf*6_bVV?Zuz6lwBH(eF%b#bjF62Yf^E&k1%V5+4c2daTRORTQ2CU6n` z<2u_-&S$sXQdQ{Po_%AoKJ(*(Q~m8@?bb?Q!(~xPFh6hhy};F5#Ti&{p?whWWnH`$ z27ZNF*YRkLGjiko`JnOGtu;2spGT8p3PfQsu|9=a}R^cSL@zYbUonP_E6AqjcZf$bt$tN|-EZytHE&x2)WW zeT+0%9r|^|89qoEe(E{o?L2;UmN$l!$bl(%;~FS|u#U#U2SV+itQw{cZ%+lIthUIE z%N*lILrbMm>vsr&`;K#p>$xKsCn$Wu9h%QGtZ}Qu+255?Ujg>|`3B{&>Ivu6L3`if zy*R30Ah!KkkfT@Y7DV6?3Ip>OetrY{<&9odWt~IZrkj=1har)bax)(p;ROK*ZhZaD z_mA;`28faR_HRYkv3g}n&I8aaQsHQw!lxvLJkWU1@`?8pwp^7acOUT&o#j9ZOJ`*YjRvhJ~({vkw3BJ^F` zq~|-*5oQMGaZ#d1hQ#4BR9fc278U4k1TH&w(3x9=rh-=J#iDVZ(E1F*SYJj|cWP6> zD%`G3H6HHj5c7HIm3FeRD*NDUi5$|bR%6b#b|o5LLCk&k>k8`y0xD@&9;Yj%_8@8M zPQbL|Vtt8?l>nA;^6J9G&qzvs&9#%m>f_AO+qi=?JWPQ#zh3t`HR+_`W(!yte@Ds` zL^&!zn`O-NOxdF1&7EtcwPEEXWk*uIsz+A#w%Gl_t`vE22w&UUD#RwY7Q5rURore_ znr{kSi9gnLgE1l?%XKON0o~u_ZM)Yg-&w{~Y=TL9`X3f;Nh|d{1m@bR zlP{3q)mpbvQP0XmKSa*{eJt!iP7Rl{IAUWR;<1Na$Vm2A;|QELr+9O_9{DV&w6u9)K?rUN?G~7VlyLY{egV!}%ZZKF=M~SxQ@eOaJaYLYJRS@OY5}vL^?O_L1JjGa%q%UR90lQichK@8HWpTEv#sfBI#dI5qjjM) z19nbeKe=2%H!1isQuA}g{JXU2xbfkMx!6duffUk{_-+X8k+>a#y`&!f4`c&KJ z(Y_#Ge%A%!gFe^NPgWbv!)qDk`z|EVVfwC9zS#2QOzdkixQw=Ra?a#r;lgTLjg3Or zocB&F1B1PZZm8&)`7z)F6z6{Au%-qCC5!qc{Zev2A-aYLb1K_6Bd4df$H%lyCe~c6 z+Nt54*E$hp?-ED6Z0u^+%j}2R$(i&4mEsFxt|rzz92|*mo69-4Fz=Xg|62lvT!n-r z)_WO`er|an`Ab(O*P)DDurcHk!nwEMw&)+&-#UMyD4*8X9n1j<3o5);t6H6-dnVdjg82* zaM84nCG`iBhoY`b2DggyIPSkJczZ^w*_^&{e~Z9Jp75Edu;Ho*=*bMZ_*%~QhON-f za9tpc^Xw>Z0cczWU$v>R7ihDW!?e>L_w=cmEdoK_(mg)O_=Qg|3lr4xlN+UgK58S~ z_DTjHTs}D=K9_PS-i%%i!%H#}iW%R~T9nmav9|fnJ&gU(s?vXB7}a z%g5xeD&Jm3wTANunk%Y;3|x6Q+Lk;G0YQhrhzzW~V}*2eSFEHIPo-2Ik?;Xea-0Ht zHO{yknWtT7(*o>(B+MAnVAh({)wYrKP zUiGMKqCUgLgNJp4qU)_4XB`X7rqeE3k2?sZk`q6@VR?bu%(~_q)kHS3ccGEOkzMI# z&&A3F?0me;^j(X$V5JcyCDc_0#sy}JQh!InZW-qqyS$FUpi9RCm)B-mLmW6DU!foh zG_iv6^#fGAAG5`i zKH78Odw5LWK;JIU+uesVZaS@0>v5K`fAj_V=l&aV}lE@tYP-7-P_+5;nBjM$Pp(w3f?nZN^)XbH8Alr{h zr+~>-168Tk!d+_1Q5gN*&w*=K-ZXVVGsxAVM;U4So_cpXLvmBpDNwPEek#P@Fxi|S zfs&>1NN%;%;rh0Db<&PWRX(s-O)J>tbgQ4fAHK8_W7HYZ%+)zmc0I z6_;N^-H-+y7~O>%J57V&$V@0_TG3-e8}Y{H88|{Lm+q9R$4ifj%g?@a5q+Ldj(qcw zEE%b>C;r;tyCWb0eqUyWbvTJJ1z!H-?NNal-n6ee1?Upzxq{>K?W#sAV1*Nq{uNrk zx7(cn158Ljh5$O);L(03rT+jU@9MK3ymS7u*WyaQfBk%fHP(FNve$FV9~}B8=M~~2 zBI>B9wdGAdNQL((Nur#l-m#YA5B=09y^?n3^KAy?+@$s|>wFnewQF&y*fnrUaUzgN z8D(<zf(Mp!jRk{bM%=Ju!i48K(e!>cZyNG6JJ}IgX9A|DN)$ zJKkO*vD|6xF1hfUlh@0;KoqDwYvJ}{-<~eeWt3oW+`H#Ey%e}+G-bW(-r{aKOH)6G zh{3fuoE?(ifb!<2@JqC}_jFJQWmbLl?KLuj`f~uz>s{u^|6*+ZPBNFX?Qw3Okohd&s!Wkz zoF%K(``?pGQfzVm_IUJ{;uUG$xk+$J#B*K7u2!vj3W)qGQ1<3zLRZa;<5-}~x1-sT zONf}SlZhN$5fV?5zW)QHaK9== z#oqpZKDhkt{Q5F&?}vEcWB7&RP0zCrkbrR5DxC4=3`Y%TBReT^UL3g<4YTf7h z!A4ir$4}A|x7!>oVjS z)D$nF9-2Hf_~%9X_xk>BDM!pV8zc;>3Pel^EjlOJ_W4oOAiOufB!zuNE_(>Wu7v7O;`% zS-;Dy-pFbmt@;e7r zu|-35NCV(u zcaM#TCCDm|)+u0Y+T%-{7dGjhJtR(tG)D|K76duDlfHF~A>Vj&HQK#|cEzj=YBl;h zAc2t7gO>hh#7JViphHX3FCSh>NOvXPZ=G{9uQykSNeoD?s9|HZ&6VmW;%c_k!YNNR z3uj65LS61mKxaJ;%&+AE?ex$)duUAq{W?><|X?^l9CHi z6uXf+lRZ7_p4fnz%7mx?vqSth;QRYNGAv?Tq|y#>*L{P4_ZAM9b4(L3k)P!n(eta0 zH&bjGN-UQi?}+a-Fimsex?1!vCdvP>u7R{xtmwnWWQ5BT1xlL2#>8XRa(vm$9#*lT zMR(rf4Q|W*{o#L3#lK0NeWvfJoJfj7H{Uu}bqFyFLbxay z>>fb-1#sanfZhhQgz-Jj&xnEWt4ZCx3Ur2#$u!q;wMRsx8yWfLysNOUkcf9)w@3<7 z3;5;~m1H1^54GBQ>h=lIT}`Sb6134^#ZCv4)_Lio*?o~`?q9D9EOw4b4=u|07J|Gh zBcW9RK8G}BeyDv!rE`qTxU&9(<1h!0q@4XgP+&>p#B`I3vY&f|W>Qz05M=HsUcpAi zd0{t}UGO6RSeiOyEkJ&FRr#C^yw~_*(tz?i$PSz^091JTB2q{Ut}XfclXk!Cg5+G2 zx3c3e`}P>z1S?In=N|9Mz>fzf&BD2~n6!ieb-lsIv5rRFu*9p0Dpg-g!!gdEXV+~D zjskJes-DG&gZD%pS&IQ&ShS-N_W)ibEG$7w%dk;%e7exB1RfSKFDhsiQ3n5}rL_XG zonc5+6Hpp^nqHOs6ZB$2b&FUpBIQ96?lBHyJT5F;*$4-%8nP6HrHlUPz!NaJRKNv> zK*nL6ofj|b!sVixo40al(b3+5Uu#X$PXU%H7>IdoUIiFO@qg&_SLeZhL-96;?O=PdCK9qs8J6iI5q8xpBr49mXX{fhhj-!$Dy;v73?m z;XkHF#^OS#b{dzpTZYJs*lcEj>1DfZ^Un+@&mF0QlCN~F)rVZipe?|Rbx`C9`Tziz zRBJi~fPw&uT>K8F0Caum&41Gb!&iIwVI*{-;$A3`*gSKiK~f^Ix5lJ(@j2XAkpnQo zseaW4Tp2BxF_7FIt`#@c9=mULH=1jM(P;4$AX{g8QW$;GBDIzM{^Y5D@iCC+IL2<_ z2hF4IZIgpcg~OZVzL#(IGZqWCRbV@cA^A+-Z7NfNi@p=f@Kb;u3;!Gvoz=fi9-2K# zxMnoLo&hpL-z`u?*Tv{7f=o8rS=8>HEH1^GrN4VN9N$JWXc$_yDfvQxOO4!-&@w#* zge@}m))6JXd3&>tRn07Za(wvStOHi%;IEHqYf@d+0TE4>he~X>^R4jc-pg$_#XYTdAWFTT#EBfPk>W?Ta9v-<{S%O_ z$xG3~d*ip|%oEnGfDFO4*=Otofx>%%@8zsJ{G5HlM#_*Vz&3%^cLu!xdr9hj5>c`% zG12#Qz)r&eSmSXLA6$MwXR-<==S`H>hngNHtt}`lNo5NZjuJv1WmyK%$*Ikf4h#-D zzA~MOA7Oqgm`RlV*s~Rebr9}$Of;Fz7CkV#@rEi;m(TU`ak{#Q=wX8P)L-q3|EK%V z`6eH3!_<0r~2 z-1NKTcDjE=4Y9K#eR>{!ZT8K){~|Nc99iV}PPVNA!t zz4f0f^+z>1A0|RoG-5S49`+@IvN>i23Yq9Sv<$^Y7!uMS?7T3v!x&CpYDBj6fByVG qtVyQ@05AcJ0Qpy#0GPA;w-QS);0cQn;LmNx|FGr%4-Ed(;r|0F?i@4# literal 0 HcmV?d00001 diff --git a/src/app.js b/src/app.js index a393df29..6309fcbd 100644 --- a/src/app.js +++ b/src/app.js @@ -8,6 +8,7 @@ const knex = require('./knex'); const fetchUpdates = require('./updates'); const fetchDeep = require('./deep'); const { storeReleases } = require('./store-releases'); +const { updateReleasesSearch } = require('./releases'); // const { storeReleaseActors } = require('./actors'); async function init() { @@ -16,6 +17,12 @@ async function init() { return; } + if (argv.updateSearch) { + await updateReleasesSearch(); + knex.destroy(); + return; + } + const updateBaseReleases = (argv.scrape || argv.sites || argv.networks) && await fetchUpdates(); const updateDeepReleases = updateBaseReleases && await fetchDeep(updateBaseReleases); diff --git a/src/releases.js b/src/releases.js index ae5dedb6..0b13c77c 100644 --- a/src/releases.js +++ b/src/releases.js @@ -368,8 +368,10 @@ async function updateReleasesSearch(releaseIds) { releases.title || ' ' || networks.name || ' ' || networks.slug || ' ' || + networks.url || ' ' || sites.name || ' ' || sites.slug || ' ' || + COALESCE(sites.url, '') || ' ' || COALESCE(sites.alias, '') || ' ' || COALESCE(releases.shoot_id, '') || ' ' || COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' || @@ -386,7 +388,7 @@ async function updateReleasesSearch(releaseIds) { LEFT JOIN tags ON local_tags.tag_id = tags.id LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''} - GROUP BY releases.id, sites.name, sites.slug, sites.alias, networks.name, networks.slug; + GROUP BY releases.id, sites.name, sites.slug, sites.alias, sites.url, networks.name, networks.slug, networks.url; `, releaseIds && [releaseIds]); if (documents.rows?.length > 0) { diff --git a/src/utils/http.js b/src/utils/http.js index ed7398d0..067839ae 100644 --- a/src/utils/http.js +++ b/src/utils/http.js @@ -1,10 +1,28 @@ 'use strict'; +const config = require('config'); +const tunnel = require('tunnel'); const bhttp = require('bhttp'); const taskQueue = require('promise-task-queue'); const logger = require('../logger')(__filename); +const proxyAgent = tunnel.httpsOverHttp({ + proxy: { + host: config.proxy.host, + port: config.proxy.port, + }, +}); + +function useProxy(url) { + if (!config.proxy.enable) { + return false; + } + + const { hostname } = new URL(url); + return config.proxy.hostnames.includes(hostname); +} + const queue = taskQueue(); queue.on('concurrencyReached:httpGet', () => { @@ -15,47 +33,42 @@ queue.on('concurrencyReached:httpPost', () => { logger.silly('Queueing POST requests'); }); -queue.define('httpGet', async ({ - url, - timeout = 30000, - options = {}, -}) => { - logger.silly(`GET ${url}`); - - const res = await bhttp.get(url, { - responseTimeout: timeout, - ...options, - }); - - res.code = res.statusCode; - - return res; -}, { - concurrency: 20, -}); - -queue.define('httpPost', async ({ +queue.define('http', async ({ url, + method = 'GET', body, timeout = 30000, options = {}, }) => { - logger.silly(`POST ${url} with ${body}`); + if (body) { + logger.silly(`${method.toUpperCase()} ${url} with ${body}`); + } else { + logger.silly(`${method.toUpperCase()} ${url}`); + } - const res = await bhttp.post(url, body, { + const reqOptions = { responseTimeout: timeout, ...options, - }); + }; - res.code = res.statusCode; + if (useProxy(url)) { + reqOptions.agent = proxyAgent; + } - return res; + const res = ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase()) + ? await bhttp[method.toLowerCase()](url, body, reqOptions) + : await bhttp[method.toLowerCase()](url, reqOptions); + + return { + ...res, + code: res.statusCode, + }; }, { concurrency: 20, }); async function get(url, options) { - return queue.push('httpGet', { + return queue.push('http', { method: 'get', url, options, @@ -63,7 +76,7 @@ async function get(url, options) { } async function post(url, body, options) { - return queue.push('httpPost', { + return queue.push('http', { url, body, options, diff --git a/src/utils/qu.js b/src/utils/qu.js index 1909a387..42691b02 100644 --- a/src/utils/qu.js +++ b/src/utils/qu.js @@ -287,7 +287,7 @@ function extractAll(htmlValue, selector) { return initAll(window.document, selector, window); } -async function get(urlValue, selector, headers, queryAll = false) { +async function get(urlValue, selector, headers, options, queryAll = false) { const res = await http.get(urlValue, { headers, }); @@ -315,8 +315,8 @@ async function get(urlValue, selector, headers, queryAll = false) { }; } -async function getAll(urlValue, selector, headers) { - return get(urlValue, selector, headers, true); +async function getAll(urlValue, selector, headers, options) { + return get(urlValue, selector, headers, options, true); } module.exports = {