add: Dockerfile for crawler

.gitignore

@@ -1,63 +1,8 @@
-built/*
-tests/cases/rwc/*
-tests/cases/perf/*
-!tests/cases/webharness/compilerToString.js
-test-args.txt
-~*.docx
-\#*\#
-.\#*
-tests/baselines/local/*
-tests/baselines/local.old/*
-tests/services/baselines/local/*
-tests/baselines/prototyping/local/*
-tests/baselines/rwc/*
-tests/baselines/reference/projectOutput/*
-tests/baselines/local/projectOutput/*
-tests/baselines/reference/testresults.tap
-tests/baselines/symlinks/*
-tests/services/baselines/prototyping/local/*
-tests/services/browser/typescriptServices.js
-src/harness/*.js
-src/compiler/diagnosticInformationMap.generated.ts
-src/compiler/diagnosticMessages.generated.json
-src/parser/diagnosticInformationMap.generated.ts
-src/parser/diagnosticMessages.generated.json
-rwc-report.html
-*.swp
-build.json
-*.actual
-tests/webTestServer.js
-tests/webTestServer.js.map
-tests/webhost/*.d.ts
-tests/webhost/webtsc.js
-tests/cases/**/*.js
-tests/cases/**/*.js.map
-*.config
-scripts/eslint/built/
-scripts/debug.bat
-scripts/run.bat
-scripts/**/*.js
-scripts/**/*.js.map
-coverage/
-internal/
 **/.DS_Store
 .settings
 **/.vs
 **/.vscode/*
-!**/.vscode/tasks.json
+
-!**/.vscode/settings.template.json
-!**/.vscode/launch.template.json
-!**/.vscode/extensions.json
-!tests/cases/projects/projectOption/**/node_modules
-!tests/cases/projects/NodeModulesSearch/**/*
-!tests/baselines/reference/project/nodeModules*/**/*
-yarn.lock
-yarn-error.log
-.parallelperf.*
-tests/baselines/reference/dt
-.failed-tests
-TEST-results.xml
-package-lock.json
 .eslintcache
 *v8.log
 
@@ -80,7 +25,6 @@ ml/filter/checkpoints
 scripts
 model/
 
-
 .astro
 
 # Database

.vscode/extensions.json

@@ -1,6 +0,0 @@
-{
-	"recommendations": [
-		"denoland.vscode-deno",
-		"bradlc.vscode-tailwindcss"
-	]
-}

.zed/settings.json

@@ -1,35 +0,0 @@
-// Folder-specific settings
-//
-// For a full list of overridable settings, and general information on folder-specific settings,
-// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
-{
-    "lsp": {
-        "deno": {
-            "settings": {
-                "deno": {
-                    "enable": true
-                }
-            }
-        }
-    },
-    "languages": {
-        "TypeScript": {
-            "language_servers": [
-                "deno",
-                "!typescript-language-server",
-                "!vtsls",
-                "!eslint"
-            ],
-            "formatter": "language_server"
-        },
-        "TSX": {
-            "language_servers": [
-                "deno",
-                "!typescript-language-server",
-                "!vtsls",
-                "!eslint"
-            ],
-            "formatter": "language_server"
-        }
-    }
-}

Dockerfile.backend

@@ -1,4 +1,4 @@
-FROM oven/bun AS bun-builder
+FROM oven/bun
 
 WORKDIR /app
 

Dockerfile.crawler

@@ -0,0 +1,13 @@
+FROM oven/bun:1.2.8-debian
+
+WORKDIR /app
+
+COPY . .
+
+RUN bun i
+
+RUN mkdir -p /app/logs
+
+WORKDIR packages/crawler
+
+CMD ["bun", "all"]

Dockerfile.frontend

@@ -1,4 +1,4 @@
-FROM oven/bun AS bun-builder
+FROM oven/bun
 
 WORKDIR /app
 

packages/core/db/pgConfig.ts

@@ -1,38 +0,0 @@
-const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT"];
-
-const unsetVars = requiredEnvVars.filter((key) => Deno.env.get(key) === undefined);
-
-if (unsetVars.length > 0) {
-	throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`);
-}
-
-const databaseHost = Deno.env.get("DB_HOST")!;
-const databaseName = Deno.env.get("DB_NAME")!;
-const databaseNameCred = Deno.env.get("DB_NAME_CRED")!;
-const databaseUser = Deno.env.get("DB_USER")!;
-const databasePassword = Deno.env.get("DB_PASSWORD")!;
-const databasePort = Deno.env.get("DB_PORT")!;
-
-export const postgresConfig = {
-	hostname: databaseHost,
-	port: parseInt(databasePort),
-	database: databaseName,
-	user: databaseUser,
-	password: databasePassword,
-};
-
-export const postgresConfigNpm = {
-	host: databaseHost,
-	port: parseInt(databasePort),
-	database: databaseName,
-	username: databaseUser,
-	password: databasePassword,
-}
-
-export const postgresConfigCred = {
-	hostname: databaseHost,
-	port: parseInt(databasePort),
-	database: databaseNameCred,
-	user: databaseUser,
-	password: databasePassword,
-};

packages/crawler/mq/index.ts

@@ -1,7 +1,14 @@
-import { Queue } from "bullmq";
+import { Queue, ConnectionOptions } from "bullmq";
+import { redis } from "@core/db/redis.ts";
 
-export const LatestVideosQueue = new Queue("latestVideos");
+export const LatestVideosQueue = new Queue("latestVideos", {
+    connection: redis as ConnectionOptions
+});
 
-export const ClassifyVideoQueue = new Queue("classifyVideo");
+export const ClassifyVideoQueue = new Queue("classifyVideo", {
+    connection: redis as ConnectionOptions
+});
 
-export const SnapshotQueue = new Queue("snapshot");
+export const SnapshotQueue = new Queue("snapshot", {
+    connection: redis as ConnectionOptions
+});

packages/crawler/package.json

@@ -1,5 +1,5 @@
 {
-	"name": "core",
+	"name": "crawler",
 	"scripts": {
 		"test": "bun --env-file=.env.test run vitest",
 		"worker:main": "bun run ./src/worker.ts",