add: Dockerfile for crawler

This commit is contained in:
alikia2x (寒寒) 2025-05-06 03:10:20 +08:00
parent 59f09ca5eb
commit 95681dcbf3
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
9 changed files with 28 additions and 143 deletions

58
.gitignore vendored
View File

@ -1,63 +1,8 @@
built/*
tests/cases/rwc/*
tests/cases/perf/*
!tests/cases/webharness/compilerToString.js
test-args.txt
~*.docx
\#*\#
.\#*
tests/baselines/local/*
tests/baselines/local.old/*
tests/services/baselines/local/*
tests/baselines/prototyping/local/*
tests/baselines/rwc/*
tests/baselines/reference/projectOutput/*
tests/baselines/local/projectOutput/*
tests/baselines/reference/testresults.tap
tests/baselines/symlinks/*
tests/services/baselines/prototyping/local/*
tests/services/browser/typescriptServices.js
src/harness/*.js
src/compiler/diagnosticInformationMap.generated.ts
src/compiler/diagnosticMessages.generated.json
src/parser/diagnosticInformationMap.generated.ts
src/parser/diagnosticMessages.generated.json
rwc-report.html
*.swp
build.json
*.actual
tests/webTestServer.js
tests/webTestServer.js.map
tests/webhost/*.d.ts
tests/webhost/webtsc.js
tests/cases/**/*.js
tests/cases/**/*.js.map
*.config
scripts/eslint/built/
scripts/debug.bat
scripts/run.bat
scripts/**/*.js
scripts/**/*.js.map
coverage/
internal/
**/.DS_Store
.settings
**/.vs
**/.vscode/*
!**/.vscode/tasks.json
!**/.vscode/settings.template.json
!**/.vscode/launch.template.json
!**/.vscode/extensions.json
!tests/cases/projects/projectOption/**/node_modules
!tests/cases/projects/NodeModulesSearch/**/*
!tests/baselines/reference/project/nodeModules*/**/*
yarn.lock
yarn-error.log
.parallelperf.*
tests/baselines/reference/dt
.failed-tests
TEST-results.xml
package-lock.json
.eslintcache
*v8.log
@ -80,7 +25,6 @@ ml/filter/checkpoints
scripts
model/
.astro
# Database

View File

@ -1,6 +0,0 @@
{
"recommendations": [
"denoland.vscode-deno",
"bradlc.vscode-tailwindcss"
]
}

View File

@ -1,35 +0,0 @@
// Folder-specific settings
//
// For a full list of overridable settings, and general information on folder-specific settings,
// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
{
"lsp": {
"deno": {
"settings": {
"deno": {
"enable": true
}
}
}
},
"languages": {
"TypeScript": {
"language_servers": [
"deno",
"!typescript-language-server",
"!vtsls",
"!eslint"
],
"formatter": "language_server"
},
"TSX": {
"language_servers": [
"deno",
"!typescript-language-server",
"!vtsls",
"!eslint"
],
"formatter": "language_server"
}
}
}

View File

@ -1,4 +1,4 @@
FROM oven/bun AS bun-builder
FROM oven/bun
WORKDIR /app

13
Dockerfile.crawler Normal file
View File

@ -0,0 +1,13 @@
FROM oven/bun:1.2.8-debian
WORKDIR /app
COPY . .
RUN bun i
RUN mkdir -p /app/logs
WORKDIR packages/crawler
CMD ["bun", "all"]

View File

@ -1,4 +1,4 @@
FROM oven/bun AS bun-builder
FROM oven/bun
WORKDIR /app

View File

@ -1,38 +0,0 @@
const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT"];
const unsetVars = requiredEnvVars.filter((key) => Deno.env.get(key) === undefined);
if (unsetVars.length > 0) {
throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`);
}
const databaseHost = Deno.env.get("DB_HOST")!;
const databaseName = Deno.env.get("DB_NAME")!;
const databaseNameCred = Deno.env.get("DB_NAME_CRED")!;
const databaseUser = Deno.env.get("DB_USER")!;
const databasePassword = Deno.env.get("DB_PASSWORD")!;
const databasePort = Deno.env.get("DB_PORT")!;
export const postgresConfig = {
hostname: databaseHost,
port: parseInt(databasePort),
database: databaseName,
user: databaseUser,
password: databasePassword,
};
export const postgresConfigNpm = {
host: databaseHost,
port: parseInt(databasePort),
database: databaseName,
username: databaseUser,
password: databasePassword,
}
export const postgresConfigCred = {
hostname: databaseHost,
port: parseInt(databasePort),
database: databaseNameCred,
user: databaseUser,
password: databasePassword,
};

View File

@ -1,7 +1,14 @@
import { Queue } from "bullmq";
import { Queue, ConnectionOptions } from "bullmq";
import { redis } from "@core/db/redis.ts";
export const LatestVideosQueue = new Queue("latestVideos");
export const LatestVideosQueue = new Queue("latestVideos", {
connection: redis as ConnectionOptions
});
export const ClassifyVideoQueue = new Queue("classifyVideo");
export const ClassifyVideoQueue = new Queue("classifyVideo", {
connection: redis as ConnectionOptions
});
export const SnapshotQueue = new Queue("snapshot");
export const SnapshotQueue = new Queue("snapshot", {
connection: redis as ConnectionOptions
});

View File

@ -1,5 +1,5 @@
{
"name": "core",
"name": "crawler",
"scripts": {
"test": "bun --env-file=.env.test run vitest",
"worker:main": "bun run ./src/worker.ts",