diff --git a/.gitignore b/.gitignore index a9cdf30..e2ff74a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,63 +1,8 @@ -built/* -tests/cases/rwc/* -tests/cases/perf/* -!tests/cases/webharness/compilerToString.js -test-args.txt -~*.docx -\#*\# -.\#* -tests/baselines/local/* -tests/baselines/local.old/* -tests/services/baselines/local/* -tests/baselines/prototyping/local/* -tests/baselines/rwc/* -tests/baselines/reference/projectOutput/* -tests/baselines/local/projectOutput/* -tests/baselines/reference/testresults.tap -tests/baselines/symlinks/* -tests/services/baselines/prototyping/local/* -tests/services/browser/typescriptServices.js -src/harness/*.js -src/compiler/diagnosticInformationMap.generated.ts -src/compiler/diagnosticMessages.generated.json -src/parser/diagnosticInformationMap.generated.ts -src/parser/diagnosticMessages.generated.json -rwc-report.html -*.swp -build.json -*.actual -tests/webTestServer.js -tests/webTestServer.js.map -tests/webhost/*.d.ts -tests/webhost/webtsc.js -tests/cases/**/*.js -tests/cases/**/*.js.map -*.config -scripts/eslint/built/ -scripts/debug.bat -scripts/run.bat -scripts/**/*.js -scripts/**/*.js.map -coverage/ -internal/ **/.DS_Store .settings **/.vs **/.vscode/* -!**/.vscode/tasks.json -!**/.vscode/settings.template.json -!**/.vscode/launch.template.json -!**/.vscode/extensions.json -!tests/cases/projects/projectOption/**/node_modules -!tests/cases/projects/NodeModulesSearch/**/* -!tests/baselines/reference/project/nodeModules*/**/* -yarn.lock -yarn-error.log -.parallelperf.* -tests/baselines/reference/dt -.failed-tests -TEST-results.xml -package-lock.json + .eslintcache *v8.log @@ -80,7 +25,6 @@ ml/filter/checkpoints scripts model/ - .astro # Database diff --git a/.vscode/extensions.json b/.vscode/extensions.json deleted file mode 100644 index ec71323..0000000 --- a/.vscode/extensions.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "recommendations": [ - "denoland.vscode-deno", - "bradlc.vscode-tailwindcss" - ] -} diff --git a/.zed/settings.json b/.zed/settings.json deleted file mode 100644 index 97f9ab8..0000000 --- a/.zed/settings.json +++ /dev/null @@ -1,35 +0,0 @@ -// Folder-specific settings -// -// For a full list of overridable settings, and general information on folder-specific settings, -// see the documentation: https://zed.dev/docs/configuring-zed#settings-files -{ - "lsp": { - "deno": { - "settings": { - "deno": { - "enable": true - } - } - } - }, - "languages": { - "TypeScript": { - "language_servers": [ - "deno", - "!typescript-language-server", - "!vtsls", - "!eslint" - ], - "formatter": "language_server" - }, - "TSX": { - "language_servers": [ - "deno", - "!typescript-language-server", - "!vtsls", - "!eslint" - ], - "formatter": "language_server" - } - } -} diff --git a/Dockerfile.backend b/Dockerfile.backend index ec04bf8..244d0b3 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -1,4 +1,4 @@ -FROM oven/bun AS bun-builder +FROM oven/bun WORKDIR /app diff --git a/Dockerfile.crawler b/Dockerfile.crawler new file mode 100644 index 0000000..4f5fe85 --- /dev/null +++ b/Dockerfile.crawler @@ -0,0 +1,13 @@ +FROM oven/bun:1.2.8-debian + +WORKDIR /app + +COPY . . + +RUN bun i + +RUN mkdir -p /app/logs + +WORKDIR packages/crawler + +CMD ["bun", "all"] \ No newline at end of file diff --git a/Dockerfile.frontend b/Dockerfile.frontend index 0580c28..8e824ab 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -1,4 +1,4 @@ -FROM oven/bun AS bun-builder +FROM oven/bun WORKDIR /app diff --git a/packages/core/db/pgConfig.ts b/packages/core/db/pgConfig.ts deleted file mode 100644 index fcc0a13..0000000 --- a/packages/core/db/pgConfig.ts +++ /dev/null @@ -1,38 +0,0 @@ -const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT"]; - -const unsetVars = requiredEnvVars.filter((key) => Deno.env.get(key) === undefined); - -if (unsetVars.length > 0) { - throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`); -} - -const databaseHost = Deno.env.get("DB_HOST")!; -const databaseName = Deno.env.get("DB_NAME")!; -const databaseNameCred = Deno.env.get("DB_NAME_CRED")!; -const databaseUser = Deno.env.get("DB_USER")!; -const databasePassword = Deno.env.get("DB_PASSWORD")!; -const databasePort = Deno.env.get("DB_PORT")!; - -export const postgresConfig = { - hostname: databaseHost, - port: parseInt(databasePort), - database: databaseName, - user: databaseUser, - password: databasePassword, -}; - -export const postgresConfigNpm = { - host: databaseHost, - port: parseInt(databasePort), - database: databaseName, - username: databaseUser, - password: databasePassword, -} - -export const postgresConfigCred = { - hostname: databaseHost, - port: parseInt(databasePort), - database: databaseNameCred, - user: databaseUser, - password: databasePassword, -}; diff --git a/packages/crawler/mq/index.ts b/packages/crawler/mq/index.ts index ef8b0f2..62d5d86 100644 --- a/packages/crawler/mq/index.ts +++ b/packages/crawler/mq/index.ts @@ -1,7 +1,14 @@ -import { Queue } from "bullmq"; +import { Queue, ConnectionOptions } from "bullmq"; +import { redis } from "@core/db/redis.ts"; -export const LatestVideosQueue = new Queue("latestVideos"); +export const LatestVideosQueue = new Queue("latestVideos", { + connection: redis as ConnectionOptions +}); -export const ClassifyVideoQueue = new Queue("classifyVideo"); +export const ClassifyVideoQueue = new Queue("classifyVideo", { + connection: redis as ConnectionOptions +}); -export const SnapshotQueue = new Queue("snapshot"); +export const SnapshotQueue = new Queue("snapshot", { + connection: redis as ConnectionOptions +}); diff --git a/packages/crawler/package.json b/packages/crawler/package.json index de74710..96abd83 100644 --- a/packages/crawler/package.json +++ b/packages/crawler/package.json @@ -1,5 +1,5 @@ { - "name": "core", + "name": "crawler", "scripts": { "test": "bun --env-file=.env.test run vitest", "worker:main": "bun run ./src/worker.ts",