feat: initial turborepo

2026-04-05 19:51:57 +02:00 · 2022-02-07 07:32:42 +07:00
parent 23528afd6e
commit 13c71e4b72
12 changed files with 537 additions and 1 deletions
--- a/package.json
+++ b/package.json
@@ -17,11 +17,16 @@
    "eslint": "^8.2.0",
    "eslint-config-google": "^0.14.0",
    "husky": "^7.0.4",
+    "turbo": "^1.1.2",
    "typescript": "^4.4.4"
  },
  "scripts": {
    "build": "echo build",
    "lint": "eslint \"+(lib|api|middleware)/**/*.ts\" --fix",
    "prepare": "husky install"
-  }
+  },
+  "workspaces": [
+	"packages/*"
+	"apps/*"
+  ]
 }
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -0,0 +1,11 @@
+{
+	"name": "tiktok-dl-core",
+	"version": "1.4.1",
+	"main": "./index.ts",
+	"license": "MIT",
+	"description": "TikTok-DL Project Core",
+	"dependencies": {
+		"got": "^11.8.2",
+		"vm2": "^3.9.5"
+	}
+}
--- a/packages/core/src/DLTikProvider.ts
+++ b/packages/core/src/DLTikProvider.ts
@@ -0,0 +1,83 @@
+import {getFetch} from '..';
+import {BaseProvider, ExtractedInfo} from './base';
+
+/**
+ * @class DLTikProvider
+ */
+export class DLTikProvider extends BaseProvider {
+  /**
+     * @return {string}
+     */
+  public resourceName(): string {
+    return 'dltik';
+  }
+
+  public client = getFetch('https://dltik.com');
+
+  public maintenance = {
+    'reason': 'My prediction is that DLTik needs an active session to use.',
+  };
+
+  /**
+   * @param {string} url - Video TikTok URL
+   * @return {Promise<ExtractedInfo>}
+   */
+  public async fetch(url: string): Promise<ExtractedInfo> {
+    // getting verification token
+    const response = await this.client('./#url=' +
+        encodeURIComponent(url));
+    const token = (
+        response.body.match(/type="hidden" value="([^""]+)"/) as string[]
+    )[1];
+
+    const dlResponse = await this.client.post('./', {
+      'form': {
+        'm': 'getlink',
+        'url': `https://m.tiktok.com/v/${
+          (/predownload\('([0-9]+)'\)/gi.exec(response.body) as string[])[1]
+        }.html`,
+        '__RequestVerificationToken': token,
+      },
+      'headers': {
+        'Origin': this.client.defaults.options.prefixUrl,
+        'Referer': response.url,
+        'Cookie': response.headers['set-cookie']?.toString(),
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'x-requested-with': 'XMLHttpRequest',
+      },
+    });
+
+    return this.extract(dlResponse.body);
+  }
+
+  /**
+   *
+   * @param {string} html - Raw
+   * @return {ExtractedInfo}
+   */
+  extract(html: string): ExtractedInfo {
+    const json = JSON.parse(html);
+    if (!json.status) {
+      return {
+        'error': json.message,
+      };
+    } else {
+    //   if (json.data.videoId === '7013188037203070234') {
+    //     return {
+    //       'error': 'Invalid url',
+    //     };
+    //   }
+      return {
+        'video': {
+          'id': json.data.videoId,
+          'urls': [json.data.watermarkVideoUrl, json.data.destinationUrl],
+          'thumb': json.data.dynamicCover,
+        },
+        'music': {
+          'url': json.data.musicUrl,
+        },
+        'caption': json.data.desc,
+      };
+    }
+  }
+};
--- a/packages/core/src/base.ts
+++ b/packages/core/src/base.ts
@@ -0,0 +1,46 @@
+import {Got} from 'got';
+
+export interface ExtractedInfo {
+    error?: string;
+    video?: {
+        id?: string;
+        thumb?: string;
+        urls: string[];
+        title?: string;
+        duration?: string;
+    };
+    music?: {
+        url: string;
+        title?: string;
+        author?: string;
+        id?: string;
+        cover?: string;
+    };
+    author?: {
+        username?: string;
+        thumb?: string;
+        id?: string;
+    };
+    caption?: string;
+    playsCount?: number;
+    sharesCount?: number;
+    commentsCount?: number;
+    likesCount?: number;
+    uploadedAt?: string;
+    updatedAt?: string;
+};
+
+export interface MaintenanceProvider {
+    reason: string;
+};
+
+/**
+ * @class BaseProvider
+ */
+export abstract class BaseProvider {
+    abstract client: Got;
+    abstract maintenance?: MaintenanceProvider;
+    abstract resourceName(): string;
+    abstract fetch(url: string): Promise<ExtractedInfo>;
+    abstract extract(html: string): ExtractedInfo;
+};
--- a/packages/core/src/dddTikProvider.ts
+++ b/packages/core/src/dddTikProvider.ts
@@ -0,0 +1,52 @@
+import {BaseProvider, ExtractedInfo} from './base';
+import {getFetch} from '..';
+import {matchLink} from './utils';
+
+/**
+ * @class DDDTikProvider
+ */
+export class DDDTikProvider extends BaseProvider {
+  /**
+     * Get resource name
+     *
+     * @return {string}
+     */
+  public resourceName(): string {
+    return 'dddtik';
+  }
+
+  public client = getFetch('https://dddtik.com');
+
+  public maintenance = undefined;
+
+  /**
+     * @param {string} url Tiktok video url
+     * @return {Promise<ExtractedInfo>}
+     */
+  async fetch(url: string): Promise<ExtractedInfo> {
+    const response = await this.client.post('./down.php', {
+      'form': {
+        'url': url,
+      },
+    });
+
+    return this.extract(response.body);
+  }
+
+  /**
+     * @param {string} html
+     * @return {ExtractedInfo}
+     */
+  extract(html: string): ExtractedInfo {
+    const urls = matchLink(html) as string[];
+    urls.pop();
+
+    const t = urls[1];
+    return {
+      'video': {
+        'urls': urls.filter((u) => u !== t),
+        'thumb': t,
+      },
+    };
+  }
+}
--- a/packages/core/src/downTikProvider.ts
+++ b/packages/core/src/downTikProvider.ts
@@ -0,0 +1,74 @@
+import {BaseProvider, ExtractedInfo} from './base';
+import {getFetch} from '..';
+import {matchCustomDownload} from './utils';
+
+/**
+ * @class DownTikProvider
+ */
+export class DownTikProvider extends BaseProvider {
+  /**
+     * Get resource name
+     *
+     * @return {string}
+     */
+  public resourceName(): string {
+    return 'downtik';
+  }
+
+  public client = getFetch('https://downtik.net');
+
+  public maintenance = undefined;
+
+  /**
+     * @param {string} url
+     *
+     * @return {Promise<ExtractedInfo>}
+     */
+  async fetch(url: string): Promise<ExtractedInfo> {
+    const response = await this.client('./');
+
+    const token = (
+            response.body.match(/id="token" value="([^""]+)"/) as string[]
+    )[1];
+
+    const responseAction = await this.client.post(
+        './action.php', {
+          'form': {
+            'url': url,
+            'token': token,
+          },
+          'headers': {
+            'cookie': response.headers['set-cookie']?.toString(),
+            'Referer': 'https://downtik.net/',
+            'Origin': 'https://downtik.net',
+          },
+        },
+    );
+
+    if (JSON.parse(responseAction.body).error) {
+      return {
+        'error': JSON.parse(responseAction.body).message,
+      };
+    };
+
+    return this.extract(JSON.parse(responseAction.body).data);
+  }
+
+  /**
+   * @param {string} html
+   * @return {ExtractedInfo}
+   */
+  extract(html: string): ExtractedInfo {
+    const urls = matchCustomDownload('downtik', html);
+
+    return {
+      'music': {
+        'url': urls.pop() as string,
+      },
+      'video': {
+        'thumb': urls?.shift(),
+        'urls': urls as string[],
+      },
+    };
+  }
+}
--- a/packages/core/src/downloaderOneProvider.ts
+++ b/packages/core/src/downloaderOneProvider.ts
@@ -0,0 +1,91 @@
+import {BaseProvider, ExtractedInfo} from './base';
+import {getFetch} from '..';
+
+/**
+ * @class DownloadOne
+ */
+export class DownloadOne extends BaseProvider {
+  /**
+     * Get provider name
+     * @return {string}
+     */
+  public resourceName(): string {
+    return 'ttdownloaderone';
+  }
+
+  public client = getFetch('http://tiktokdownloader.one');
+
+  public maintenance = undefined;
+
+  /**
+     * Fetch ttdownloader.one
+     * @param {string} url Video TikTok URL
+     * @return {Promise<ExtractedInfo>}
+     */
+  public async fetch(
+      url: string,
+  ): Promise<ExtractedInfo> {
+    // getting the token
+    const response = await this.client('./');
+
+    const token = (/name="_token_" content="(.*)"/gi
+        .exec(response.body) as string[])[1];
+
+    const dlResponse = await this.client(
+        './api/v1/fetch?url=' + url, {
+          'headers': {
+            'TOKEN': token,
+            'Referer': 'http://tiktokdownloader.one/',
+            'Origin': 'http://tiktokdownloader.one',
+            'Accept': 'application/json, text/plain, */*',
+          },
+        },
+    );
+
+    if (dlResponse.statusCode !== 200) {
+      return {
+        'error': 'Probably the video doesn\'t exist',
+      };
+    }
+
+    return this.extract(dlResponse.body);
+  }
+
+  /**
+     * Extract page from ttdownloader.one site
+     * @param {string} html
+     * @return {ExtractedInfo}
+     */
+  extract(html: string): ExtractedInfo {
+    const json = JSON.parse(html);
+
+    return {
+      'video': {
+        'urls': [
+          json.url,
+          json.url_nwm,
+        ],
+        'thumb': json.cover,
+        'id': json.video_id,
+      },
+      'music': {
+        'url': json.music.url,
+        'title': json.music.title,
+        'cover': json.music.cover,
+        'author': json.music.author,
+      },
+      'author': {
+        'id': json.user.name,
+        'username': json.user.username,
+        'thumb': json.user.cover,
+      },
+      'caption': json.caption,
+      'updatedAt': json.updatedAt ?? '-',
+      'uploadedAt': json.uploaded_at,
+      'commentsCount': json.stats.comment,
+      'sharesCount': json.stats.shares,
+      'likesCount': json.stats.likes,
+      'playsCount': json.stats.play,
+    };
+  }
+}
--- a/packages/core/src/utils/extractor.ts
+++ b/packages/core/src/utils/extractor.ts
@@ -0,0 +1,73 @@
+import {getProvider} from '..';
+import type {BaseProvider} from '../baseProvider';
+
+import {NodeVM} from 'vm2';
+
+export const deObfuscate = (html: string): string => {
+  if (/error/gi.test(html)) {
+    throw new Error(html.split('\'')
+        .find((x) => /(((url)? error)|could)/gi.test(x)),
+    );
+  } else {
+    // only match script tag
+    const obfuscatedScripts = html
+        .match(/<script[\s\S]*?>[\s\S]*?<\/script>/gi);
+    if (!obfuscatedScripts?.length) {
+      throw new Error(
+          'Cannot download the video!',
+      );
+    } else {
+      const transformed = obfuscatedScripts[0]
+          .replace(/<(\/)?script( type=".+")?>/g, '').trim().replace('eval', '')
+          .replace(/\(function(.)?\(h/gi, 'module.exports = (function (h');
+      const deObfuscated = new NodeVM({
+        'compiler': 'javascript',
+        'console': 'inherit',
+        'require': {
+          'external': true,
+          'root': './',
+        },
+      }).run(transformed, 'deobfuscate.js');
+      return deObfuscated;
+    }
+  }
+};
+
+
+export const matchLink = (raw: string): string[] | null => {
+  // eslint-disable-next-line max-len
+  return raw.match(/https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/gi);
+};
+
+export const matchCustomDownload =
+(provider: string, raw: string): string[] => {
+  const links = matchLink(raw) as string[];
+  const urls = raw.match(/\/download.php\?token=(.*?)"/gi)
+      ?.map((url) => (getProvider(provider) as BaseProvider).client.
+          defaults.options.prefixUrl.slice(0, -1)+
+      url.slice(0, -3));
+
+  return [links[0]].concat(urls as string[]);
+};
+
+export const deObfuscateSaveFromScript = (scriptContent: string): string => {
+  const safeScript = 'let result;' +
+  scriptContent.replace(/\/\*js\-response\*\//gi, '')
+      .replace(/eval\(a\)/gi, 'return a')
+      .replace(/\[\]\["filter"\]\["constructor"\]\(b\)\.call\(a\);/gi, `
+        if (b.includes('showResult')) {
+          result = b;
+          return;
+        } else []['filter']['constructo'](b).call(a);`) +
+         'module.exports = result;';
+  const vm = new NodeVM({
+    'compiler': 'javascript',
+    'console': 'inherit',
+    'require': {
+      'external': true,
+      'root': './',
+    },
+  });
+  const result = vm.run(safeScript, 'savefrom.js');
+  return result;
+};r
--- a/packages/core/src/utils/generator.ts
+++ b/packages/core/src/utils/generator.ts
@@ -0,0 +1,12 @@
+/**
+ * Generate key for ttsave.app
+ *
+ * @param {string} token - Generated token by TTSave
+ * @return {string | undefined}
+ */
+export const keyGeneratorTTSave = async (token: string): Promise<string> => {
+  const expectedLen = (token.length / 3);
+
+  return token.split('').reverse().join('')
+      .slice(-(expectedLen));
+};
--- a/packages/core/src/utils/index.ts
+++ b/packages/core/src/utils/index.ts
@@ -0,0 +1,2 @@
+export * from './generator';
+export * from './extractor';
--- a/turbo.json
+++ b/turbo.json
@@ -0,0 +1,9 @@
+{
+	"baseBranch": "origin/main",
+	"pipeline": {
+		"build": {
+			"dependsOn": ["^build"]
+		},
+		"lint": {}
+	}
+}
--- a/yarn.lock
+++ b/yarn.lock
@@ -1150,6 +1150,84 @@ tsutils@^3.21.0:
  dependencies:
    tslib "^1.8.1"

+turbo-darwin-64@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-darwin-64/-/turbo-darwin-64-1.1.2.tgz#b129aaf538821de78d5e2129495c553627174650"
+  integrity sha512-rua17HnVvAqAU54gVfiQoH7cfopOqANv+yI6NtxLMD8aFfX2cJ9m8SSvH2v2vCaToNDW6OnTkdqDKQpqIHzbCw==
+
+turbo-darwin-arm64@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-darwin-arm64/-/turbo-darwin-arm64-1.1.2.tgz#07a783ad2e3e8af600ae7406cc4062ff56ac0351"
+  integrity sha512-otqSQNYDyKg0KqB3NM0BI4oiRPKdJkUE/XBn8dcUS+zeRLrL00XtaM0eSwynZs1tb6zU/Y+SPMSBRygD1TCOnw==
+
+turbo-freebsd-64@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-freebsd-64/-/turbo-freebsd-64-1.1.2.tgz#9e22abf04ec2298f205a57b5c9ce14e22844baf3"
+  integrity sha512-2nxwVDTAM0DtIQ2i3UOfEsQLF7vp+XZ/b9SKtiHxz710fXvdyuGivYI25axDdcBn8kQ45rnbUnarF1aW8CMGgg==
+
+turbo-freebsd-arm64@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-freebsd-arm64/-/turbo-freebsd-arm64-1.1.2.tgz#6095c9012881225a5fdfb55362defa12f24b1f8e"
+  integrity sha512-ro1Ah96yzgzyT0BJe1mceAqxPxi0pUwzAvN3IKVpMqi4hYkT3aRbzDCaSxzyC6let2Al/NUsgHnbAv38OF2Xkw==
+
+turbo-linux-32@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-linux-32/-/turbo-linux-32-1.1.2.tgz#4726e533d6966172b6bc4a960524ec2eb61adaab"
+  integrity sha512-HKBsETxQMVaf/DJwMg7pypPbGA6KEu0gEf9C8o2aPJvwMPBYgNsNaU08Xizuh5xzEQTzpbIWfQyqdNgMV4RG3Q==
+
+turbo-linux-64@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-linux-64/-/turbo-linux-64-1.1.2.tgz#dfe7f3a4c91acecdb84ecab330acee06857e568e"
+  integrity sha512-IklKsOklcRHIWkTzKg95BQ6jgJ53kLvRMrp8yqzlvZprkWdiyhAgUxrUTTHOOTce2XA3+jdN2+MwixG44uY2vg==
+
+turbo-linux-arm64@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-linux-arm64/-/turbo-linux-arm64-1.1.2.tgz#c39b6c50657fa0e82627407c86a5c43f19598e2b"
+  integrity sha512-3kS6sk2lOtuBBqkcL+yeGqD1yew4UZ1o7XUcbDD8UPwhF2kAfK7Qs0vTJw4lnO1scjhihkoTrmXM7yozvjf4/w==
+
+turbo-linux-arm@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-linux-arm/-/turbo-linux-arm-1.1.2.tgz#2f51f93a3aa144b8ba25d7b0e3c53ea186a0e9dd"
+  integrity sha512-CNbaTvRozq7H/5jpy9OZlzJ6BkeEXF+nF2n9dHiUrbAXd3nq84Qt9odcQJmGnexP19YS9w6l3tIHncX4BgwtqA==
+
+turbo-linux-mips64le@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-linux-mips64le/-/turbo-linux-mips64le-1.1.2.tgz#f52b7f410ac289d4e539f108679d2324aa5e271e"
+  integrity sha512-CDoXVIlW43C6KLgYxe13KkG8h6DswXHxbTVHiZdOwRQ56j46lU+JOVpLoh6wpQGcHvj58VEiypZBRTGVFMeogw==
+
+turbo-linux-ppc64le@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-linux-ppc64le/-/turbo-linux-ppc64le-1.1.2.tgz#18d08d3414075d0dcb4be83ca837dda508313996"
+  integrity sha512-xPVMHoiOJE/qI63jSOXwYIUFQXLdstxDV6fLnRxvq0QnJNxgTKq+mLUeE8M4LDVh1bdqHLcfk/HmyQ6+X1XVkQ==
+
+turbo-windows-32@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-windows-32/-/turbo-windows-32-1.1.2.tgz#96033019094bcb091647d6063c3c9b8e83d0acbe"
+  integrity sha512-Gj1yvPE0aMDSOxGVSBaecLnwsVDT1xX8U0dtLrg52TYY2jlaci0atjHKr9nTFuX7z8uwAf6PopwdriGoCeT3ng==
+
+turbo-windows-64@1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo-windows-64/-/turbo-windows-64-1.1.2.tgz#8eb3f77ab7e04b077752ae2204114c82e5c74697"
+  integrity sha512-0Ncx/iKhnKrdAU8hJ+8NUcF9jtFr8KoW5mMWfiFzy+mgUbVKbpzWT2eoGR6zJExedQsRvYOejbEX5iihbnj5bA==
+
+turbo@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/turbo/-/turbo-1.1.2.tgz#751b9651dc3ebe469898db76afab6405666ad0ff"
+  integrity sha512-3ViHKyAkaBKNKwHASTa1zkVT3tVVhQNLrpxBS7LoN+794ouQUYmy6lf0rTqzG3iTZHtIDwC+piZSdTl4XjEVMg==
+  optionalDependencies:
+    turbo-darwin-64 "1.1.2"
+    turbo-darwin-arm64 "1.1.2"
+    turbo-freebsd-64 "1.1.2"
+    turbo-freebsd-arm64 "1.1.2"
+    turbo-linux-32 "1.1.2"
+    turbo-linux-64 "1.1.2"
+    turbo-linux-arm "1.1.2"
+    turbo-linux-arm64 "1.1.2"
+    turbo-linux-mips64le "1.1.2"
+    turbo-linux-ppc64le "1.1.2"
+    turbo-windows-32 "1.1.2"
+    turbo-windows-64 "1.1.2"
+
 type-check@^0.4.0, type-check@~0.4.0:
  version "0.4.0"
  resolved "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz"