add: Dockerfile for crawler
This commit is contained in:
parent
59f09ca5eb
commit
95681dcbf3
58
.gitignore
vendored
58
.gitignore
vendored
@ -1,63 +1,8 @@
|
||||
built/*
|
||||
tests/cases/rwc/*
|
||||
tests/cases/perf/*
|
||||
!tests/cases/webharness/compilerToString.js
|
||||
test-args.txt
|
||||
~*.docx
|
||||
\#*\#
|
||||
.\#*
|
||||
tests/baselines/local/*
|
||||
tests/baselines/local.old/*
|
||||
tests/services/baselines/local/*
|
||||
tests/baselines/prototyping/local/*
|
||||
tests/baselines/rwc/*
|
||||
tests/baselines/reference/projectOutput/*
|
||||
tests/baselines/local/projectOutput/*
|
||||
tests/baselines/reference/testresults.tap
|
||||
tests/baselines/symlinks/*
|
||||
tests/services/baselines/prototyping/local/*
|
||||
tests/services/browser/typescriptServices.js
|
||||
src/harness/*.js
|
||||
src/compiler/diagnosticInformationMap.generated.ts
|
||||
src/compiler/diagnosticMessages.generated.json
|
||||
src/parser/diagnosticInformationMap.generated.ts
|
||||
src/parser/diagnosticMessages.generated.json
|
||||
rwc-report.html
|
||||
*.swp
|
||||
build.json
|
||||
*.actual
|
||||
tests/webTestServer.js
|
||||
tests/webTestServer.js.map
|
||||
tests/webhost/*.d.ts
|
||||
tests/webhost/webtsc.js
|
||||
tests/cases/**/*.js
|
||||
tests/cases/**/*.js.map
|
||||
*.config
|
||||
scripts/eslint/built/
|
||||
scripts/debug.bat
|
||||
scripts/run.bat
|
||||
scripts/**/*.js
|
||||
scripts/**/*.js.map
|
||||
coverage/
|
||||
internal/
|
||||
**/.DS_Store
|
||||
.settings
|
||||
**/.vs
|
||||
**/.vscode/*
|
||||
!**/.vscode/tasks.json
|
||||
!**/.vscode/settings.template.json
|
||||
!**/.vscode/launch.template.json
|
||||
!**/.vscode/extensions.json
|
||||
!tests/cases/projects/projectOption/**/node_modules
|
||||
!tests/cases/projects/NodeModulesSearch/**/*
|
||||
!tests/baselines/reference/project/nodeModules*/**/*
|
||||
yarn.lock
|
||||
yarn-error.log
|
||||
.parallelperf.*
|
||||
tests/baselines/reference/dt
|
||||
.failed-tests
|
||||
TEST-results.xml
|
||||
package-lock.json
|
||||
|
||||
.eslintcache
|
||||
*v8.log
|
||||
|
||||
@ -80,7 +25,6 @@ ml/filter/checkpoints
|
||||
scripts
|
||||
model/
|
||||
|
||||
|
||||
.astro
|
||||
|
||||
# Database
|
||||
|
6
.vscode/extensions.json
vendored
6
.vscode/extensions.json
vendored
@ -1,6 +0,0 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"denoland.vscode-deno",
|
||||
"bradlc.vscode-tailwindcss"
|
||||
]
|
||||
}
|
@ -1,35 +0,0 @@
|
||||
// Folder-specific settings
|
||||
//
|
||||
// For a full list of overridable settings, and general information on folder-specific settings,
|
||||
// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
|
||||
{
|
||||
"lsp": {
|
||||
"deno": {
|
||||
"settings": {
|
||||
"deno": {
|
||||
"enable": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"languages": {
|
||||
"TypeScript": {
|
||||
"language_servers": [
|
||||
"deno",
|
||||
"!typescript-language-server",
|
||||
"!vtsls",
|
||||
"!eslint"
|
||||
],
|
||||
"formatter": "language_server"
|
||||
},
|
||||
"TSX": {
|
||||
"language_servers": [
|
||||
"deno",
|
||||
"!typescript-language-server",
|
||||
"!vtsls",
|
||||
"!eslint"
|
||||
],
|
||||
"formatter": "language_server"
|
||||
}
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
FROM oven/bun AS bun-builder
|
||||
FROM oven/bun
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
13
Dockerfile.crawler
Normal file
13
Dockerfile.crawler
Normal file
@ -0,0 +1,13 @@
|
||||
FROM oven/bun:1.2.8-debian
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN bun i
|
||||
|
||||
RUN mkdir -p /app/logs
|
||||
|
||||
WORKDIR packages/crawler
|
||||
|
||||
CMD ["bun", "all"]
|
@ -1,4 +1,4 @@
|
||||
FROM oven/bun AS bun-builder
|
||||
FROM oven/bun
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
@ -1,38 +0,0 @@
|
||||
const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT"];
|
||||
|
||||
const unsetVars = requiredEnvVars.filter((key) => Deno.env.get(key) === undefined);
|
||||
|
||||
if (unsetVars.length > 0) {
|
||||
throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`);
|
||||
}
|
||||
|
||||
const databaseHost = Deno.env.get("DB_HOST")!;
|
||||
const databaseName = Deno.env.get("DB_NAME")!;
|
||||
const databaseNameCred = Deno.env.get("DB_NAME_CRED")!;
|
||||
const databaseUser = Deno.env.get("DB_USER")!;
|
||||
const databasePassword = Deno.env.get("DB_PASSWORD")!;
|
||||
const databasePort = Deno.env.get("DB_PORT")!;
|
||||
|
||||
export const postgresConfig = {
|
||||
hostname: databaseHost,
|
||||
port: parseInt(databasePort),
|
||||
database: databaseName,
|
||||
user: databaseUser,
|
||||
password: databasePassword,
|
||||
};
|
||||
|
||||
export const postgresConfigNpm = {
|
||||
host: databaseHost,
|
||||
port: parseInt(databasePort),
|
||||
database: databaseName,
|
||||
username: databaseUser,
|
||||
password: databasePassword,
|
||||
}
|
||||
|
||||
export const postgresConfigCred = {
|
||||
hostname: databaseHost,
|
||||
port: parseInt(databasePort),
|
||||
database: databaseNameCred,
|
||||
user: databaseUser,
|
||||
password: databasePassword,
|
||||
};
|
@ -1,7 +1,14 @@
|
||||
import { Queue } from "bullmq";
|
||||
import { Queue, ConnectionOptions } from "bullmq";
|
||||
import { redis } from "@core/db/redis.ts";
|
||||
|
||||
export const LatestVideosQueue = new Queue("latestVideos");
|
||||
export const LatestVideosQueue = new Queue("latestVideos", {
|
||||
connection: redis as ConnectionOptions
|
||||
});
|
||||
|
||||
export const ClassifyVideoQueue = new Queue("classifyVideo");
|
||||
export const ClassifyVideoQueue = new Queue("classifyVideo", {
|
||||
connection: redis as ConnectionOptions
|
||||
});
|
||||
|
||||
export const SnapshotQueue = new Queue("snapshot");
|
||||
export const SnapshotQueue = new Queue("snapshot", {
|
||||
connection: redis as ConnectionOptions
|
||||
});
|
||||
|
@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "core",
|
||||
"name": "crawler",
|
||||
"scripts": {
|
||||
"test": "bun --env-file=.env.test run vitest",
|
||||
"worker:main": "bun run ./src/worker.ts",
|
||||
|
Loading…
Reference in New Issue
Block a user