From 04f51cf988ffee3aba75f720f9880b10be5644ca Mon Sep 17 00:00:00 2001 From: hansputera Date: Sun, 8 Sep 2024 09:34:56 +0800 Subject: [PATCH] fix(packages.core): fix musicalydown urls not found, and 403 forbidden issue Signed-off-by: hansputera --- apps/web/lib/rotator.ts | 1 - packages/core/src/getVidTikProvider.ts | 117 ---------------------- packages/core/src/index.ts | 8 +- packages/core/src/loveTikProvider.ts | 74 -------------- packages/core/src/musicalyDownProvider.ts | 20 ++-- 5 files changed, 13 insertions(+), 207 deletions(-) delete mode 100644 packages/core/src/getVidTikProvider.ts delete mode 100644 packages/core/src/loveTikProvider.ts diff --git a/apps/web/lib/rotator.ts b/apps/web/lib/rotator.ts index 5157be8..776f9ea 100644 --- a/apps/web/lib/rotator.ts +++ b/apps/web/lib/rotator.ts @@ -37,7 +37,6 @@ export const rotateProvider = async ( if (!cachedData) { try { const data = await provider.fetch(url, params ?? {}); - console.log(data); if (data.error) { if (!skipOnError) { return { diff --git a/packages/core/src/getVidTikProvider.ts b/packages/core/src/getVidTikProvider.ts deleted file mode 100644 index 998d472..0000000 --- a/packages/core/src/getVidTikProvider.ts +++ /dev/null @@ -1,117 +0,0 @@ -import {BaseProvider, ExtractedInfo} from './base'; -import {getFetch} from '../fetch'; -import {matchLink} from './utils'; -import {random as randomUA} from 'tiktok-dl-config/useragents'; -import type {Shape} from 'ow'; - -/** - * @class GetVidTikProvider - */ -export class GetVidTikProvider extends BaseProvider { - /** - * Get resource name - * - * @return {string} - */ - public resourceName(): string { - return 'getvidtik'; - } - - public client = getFetch('https://getvidtik.com'); - public maintenance = undefined; - - /** - * @param {string} url TikTok Video URL - * @return {Promise} - */ - async fetch(url: string): Promise { - // getting the token. - const response = await this.client.get('./', { - headers: { - 'User-Agent': randomUA(), - }, - }); - - const matchs = /name="token" type="hidden" value="([^""]+)"/.exec( - response.body, - ); - if (!matchs) { - return { - error: "Couldn't get the token.", - }; - } else { - // download request - const downloadResponse = await this.client.post('./download', { - headers: { - 'User-Agent': randomUA(), - Cookie: response.headers['set-cookie']?.toString(), - Origin: this.client.defaults.options.prefixUrl.toString(), - Referer: response.url, - }, - followRedirect: false, - form: { - url, - token: matchs[1], - }, - }); - - if (downloadResponse.statusCode === 302) { - return { - error: 'The video is private or removed. Please try again!', - }; - } else { - return this.extract(downloadResponse.body); - } - } - } - - /** - * Extract getVidTik Response - * @param {string} html - Raw - * @return {ExtractedInfo} - */ - extract(html: string): ExtractedInfo { - const matchs = matchLink(html); - if (matchs) { - const tiktokMatchs = matchs.filter((url) => - /http(s)?:\/\/(.*)\.(tiktok(cdn)?\.com|akamaized.net)/gi.test(url), - ); - - if (tiktokMatchs) { - const metadataMatchs = html - .match(/(.+)<\/td>/)![0] - .split(/<.*?>/g) - .filter((x) => x.length); - return { - video: { - thumb: tiktokMatchs[0], - urls: [tiktokMatchs[2], tiktokMatchs[3]], // [0] = no watermark, [1] = watermark - }, - music: { - url: tiktokMatchs[tiktokMatchs.length - 1], // soon, i'll use '.at(-1)' - }, - caption: metadataMatchs[3], - author: { - nick: metadataMatchs[1], - }, - }; - } else { - return { - error: "Couldn't match tiktok links.", - }; - } - } else { - return { - error: "Couldn't match site links.", - }; - } - } - - /** - * Get ow.Shape params. - * @return {Shape | undefined} - */ - public getParams(): Shape | undefined { - return undefined; - } -} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8385baf..5d3ed2f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -8,11 +8,11 @@ import {SaveFromProvider} from './saveFromProvider'; import {SaveTikProvider} from './saveTikProvider'; import {TikDownProvider} from './tikDownProvider'; import {DownTikProvider} from './downTikProvider'; -import {LoveTikProvider} from './loveTikProvider'; +// import {LoveTikProvider} from './loveTikProvider'; // import {DDDTikProvider} from './dddTikProvider'; // import {DownloadOne} from './downloaderOneProvider'; import {NativeProvider} from './nativeProvider'; -import {GetVidTikProvider} from './getVidTikProvider'; +// import {GetVidTikProvider} from './getVidTikProvider'; export const Providers: BaseProvider[] = [ new SnaptikProvider(), @@ -23,11 +23,11 @@ export const Providers: BaseProvider[] = [ new SaveTikProvider(), new TikDownProvider(), new DownTikProvider(), // SaveTik Mirror - new LoveTikProvider(), + // new LoveTikProvider(), // new DDDTikProvider(), // new DownloadOne(), new NativeProvider(), - new GetVidTikProvider(), + // new GetVidTikProvider(), ]; export const getRandomProvider = () => diff --git a/packages/core/src/loveTikProvider.ts b/packages/core/src/loveTikProvider.ts deleted file mode 100644 index 2124ebd..0000000 --- a/packages/core/src/loveTikProvider.ts +++ /dev/null @@ -1,74 +0,0 @@ -import {BaseProvider, ExtractedInfo} from './base'; -import {getFetch} from '../fetch'; -import type {Shape} from 'ow'; - -/** - * @class LoveTikProvider - */ -export class LoveTikProvider extends BaseProvider { - /** - * Get resource name - * - * @return {string} - */ - public resourceName(): string { - return 'lovetik'; - } - - public client = getFetch('https://lovetik.com'); - - public maintenance = undefined; - - /** - * @param {string} url Video TikTok URL - * @return {Promise} - */ - async fetch(url: string): Promise { - const response = await this.client.post('./api/ajax/search', { - form: { - query: url, - }, - headers: { - Origin: 'https://lovetik.com/', - Referer: 'https://lovetik.com/', - }, - }); - - return this.extract(response.body); - } - - /** - * @param {string} jsonString - * @return {ExtractedInfo} - */ - extract(jsonString: string): ExtractedInfo { - const json = JSON.parse(jsonString); - - if (json.mess) { - return { - error: json.mess, - }; - } - - return { - music: { - url: json.links.pop().a, - }, - video: { - thumb: json.cover, - urls: json.links.map((l: Record) => l.a), - }, - author: { - username: json.author.replace(/(<([^>]+)>)/gi, ''), - }, - }; - } - - /** - * Get ow.Shape params. - * @return {Shape | undefined} - */ - public getParams(): Shape | undefined { - return undefined; - } -} diff --git a/packages/core/src/musicalyDownProvider.ts b/packages/core/src/musicalyDownProvider.ts index 9494ad4..9b0f276 100644 --- a/packages/core/src/musicalyDownProvider.ts +++ b/packages/core/src/musicalyDownProvider.ts @@ -1,6 +1,7 @@ import {getFetch} from '../fetch'; import {BaseProvider, ExtractedInfo} from './base'; import type {Shape} from 'ow'; +import { matchLink } from './utils'; /** * @class MusicalyDown @@ -28,11 +29,14 @@ export class MusicalyDown extends BaseProvider { Accept: '*/*', Referer: this.client.defaults.options.prefixUrl.toString(), Origin: this.client.defaults.options.prefixUrl.toString(), + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36' }, }); + const tokens = res.body.match( /input name="([^""]+)" type="hidden" value="([^""]+)"/, ) as string[]; + const response = await this.client.post('./download', { form: { [(res.body.match(/input name="([^"]+)/) as string[])[1]]: url, @@ -44,6 +48,7 @@ export class MusicalyDown extends BaseProvider { Accept: '*/*', Referer: this.client.defaults.options.prefixUrl.toString(), Origin: this.client.defaults.options.prefixUrl.toString(), + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', }, }); @@ -56,19 +61,12 @@ export class MusicalyDown extends BaseProvider { * @return {ExtractedInfo} */ public extract(html: string): ExtractedInfo { - const matchUrls = html.match( - //gi, - ) as string[]; - const urls = matchUrls.map( - (url) => - //gi.exec( - url, - )?.[1] as string, - ); + const urls = matchLink(html); + return { video: { - urls: urls, - thumb: /img class="responsive-img" src="(.*?)"/gi.exec( + urls: urls?.filter(url => /muscdn/gi.test(url)) ?? [], + thumb: /img src="(.*?)" alt class="circle responsive-img"/gi.exec( html, )?.[1], },