add: Dockerfile for crawler
This commit is contained in:
parent
59f09ca5eb
commit
95681dcbf3
58
.gitignore
vendored
58
.gitignore
vendored
@ -1,63 +1,8 @@
|
|||||||
built/*
|
|
||||||
tests/cases/rwc/*
|
|
||||||
tests/cases/perf/*
|
|
||||||
!tests/cases/webharness/compilerToString.js
|
|
||||||
test-args.txt
|
|
||||||
~*.docx
|
|
||||||
\#*\#
|
|
||||||
.\#*
|
|
||||||
tests/baselines/local/*
|
|
||||||
tests/baselines/local.old/*
|
|
||||||
tests/services/baselines/local/*
|
|
||||||
tests/baselines/prototyping/local/*
|
|
||||||
tests/baselines/rwc/*
|
|
||||||
tests/baselines/reference/projectOutput/*
|
|
||||||
tests/baselines/local/projectOutput/*
|
|
||||||
tests/baselines/reference/testresults.tap
|
|
||||||
tests/baselines/symlinks/*
|
|
||||||
tests/services/baselines/prototyping/local/*
|
|
||||||
tests/services/browser/typescriptServices.js
|
|
||||||
src/harness/*.js
|
|
||||||
src/compiler/diagnosticInformationMap.generated.ts
|
|
||||||
src/compiler/diagnosticMessages.generated.json
|
|
||||||
src/parser/diagnosticInformationMap.generated.ts
|
|
||||||
src/parser/diagnosticMessages.generated.json
|
|
||||||
rwc-report.html
|
|
||||||
*.swp
|
|
||||||
build.json
|
|
||||||
*.actual
|
|
||||||
tests/webTestServer.js
|
|
||||||
tests/webTestServer.js.map
|
|
||||||
tests/webhost/*.d.ts
|
|
||||||
tests/webhost/webtsc.js
|
|
||||||
tests/cases/**/*.js
|
|
||||||
tests/cases/**/*.js.map
|
|
||||||
*.config
|
|
||||||
scripts/eslint/built/
|
|
||||||
scripts/debug.bat
|
|
||||||
scripts/run.bat
|
|
||||||
scripts/**/*.js
|
|
||||||
scripts/**/*.js.map
|
|
||||||
coverage/
|
|
||||||
internal/
|
|
||||||
**/.DS_Store
|
**/.DS_Store
|
||||||
.settings
|
.settings
|
||||||
**/.vs
|
**/.vs
|
||||||
**/.vscode/*
|
**/.vscode/*
|
||||||
!**/.vscode/tasks.json
|
|
||||||
!**/.vscode/settings.template.json
|
|
||||||
!**/.vscode/launch.template.json
|
|
||||||
!**/.vscode/extensions.json
|
|
||||||
!tests/cases/projects/projectOption/**/node_modules
|
|
||||||
!tests/cases/projects/NodeModulesSearch/**/*
|
|
||||||
!tests/baselines/reference/project/nodeModules*/**/*
|
|
||||||
yarn.lock
|
|
||||||
yarn-error.log
|
|
||||||
.parallelperf.*
|
|
||||||
tests/baselines/reference/dt
|
|
||||||
.failed-tests
|
|
||||||
TEST-results.xml
|
|
||||||
package-lock.json
|
|
||||||
.eslintcache
|
.eslintcache
|
||||||
*v8.log
|
*v8.log
|
||||||
|
|
||||||
@ -80,7 +25,6 @@ ml/filter/checkpoints
|
|||||||
scripts
|
scripts
|
||||||
model/
|
model/
|
||||||
|
|
||||||
|
|
||||||
.astro
|
.astro
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
|
6
.vscode/extensions.json
vendored
6
.vscode/extensions.json
vendored
@ -1,6 +0,0 @@
|
|||||||
{
|
|
||||||
"recommendations": [
|
|
||||||
"denoland.vscode-deno",
|
|
||||||
"bradlc.vscode-tailwindcss"
|
|
||||||
]
|
|
||||||
}
|
|
@ -1,35 +0,0 @@
|
|||||||
// Folder-specific settings
|
|
||||||
//
|
|
||||||
// For a full list of overridable settings, and general information on folder-specific settings,
|
|
||||||
// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
|
|
||||||
{
|
|
||||||
"lsp": {
|
|
||||||
"deno": {
|
|
||||||
"settings": {
|
|
||||||
"deno": {
|
|
||||||
"enable": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"languages": {
|
|
||||||
"TypeScript": {
|
|
||||||
"language_servers": [
|
|
||||||
"deno",
|
|
||||||
"!typescript-language-server",
|
|
||||||
"!vtsls",
|
|
||||||
"!eslint"
|
|
||||||
],
|
|
||||||
"formatter": "language_server"
|
|
||||||
},
|
|
||||||
"TSX": {
|
|
||||||
"language_servers": [
|
|
||||||
"deno",
|
|
||||||
"!typescript-language-server",
|
|
||||||
"!vtsls",
|
|
||||||
"!eslint"
|
|
||||||
],
|
|
||||||
"formatter": "language_server"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,4 +1,4 @@
|
|||||||
FROM oven/bun AS bun-builder
|
FROM oven/bun
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
13
Dockerfile.crawler
Normal file
13
Dockerfile.crawler
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
FROM oven/bun:1.2.8-debian
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN bun i
|
||||||
|
|
||||||
|
RUN mkdir -p /app/logs
|
||||||
|
|
||||||
|
WORKDIR packages/crawler
|
||||||
|
|
||||||
|
CMD ["bun", "all"]
|
@ -1,4 +1,4 @@
|
|||||||
FROM oven/bun AS bun-builder
|
FROM oven/bun
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
@ -1,38 +0,0 @@
|
|||||||
const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT"];
|
|
||||||
|
|
||||||
const unsetVars = requiredEnvVars.filter((key) => Deno.env.get(key) === undefined);
|
|
||||||
|
|
||||||
if (unsetVars.length > 0) {
|
|
||||||
throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const databaseHost = Deno.env.get("DB_HOST")!;
|
|
||||||
const databaseName = Deno.env.get("DB_NAME")!;
|
|
||||||
const databaseNameCred = Deno.env.get("DB_NAME_CRED")!;
|
|
||||||
const databaseUser = Deno.env.get("DB_USER")!;
|
|
||||||
const databasePassword = Deno.env.get("DB_PASSWORD")!;
|
|
||||||
const databasePort = Deno.env.get("DB_PORT")!;
|
|
||||||
|
|
||||||
export const postgresConfig = {
|
|
||||||
hostname: databaseHost,
|
|
||||||
port: parseInt(databasePort),
|
|
||||||
database: databaseName,
|
|
||||||
user: databaseUser,
|
|
||||||
password: databasePassword,
|
|
||||||
};
|
|
||||||
|
|
||||||
export const postgresConfigNpm = {
|
|
||||||
host: databaseHost,
|
|
||||||
port: parseInt(databasePort),
|
|
||||||
database: databaseName,
|
|
||||||
username: databaseUser,
|
|
||||||
password: databasePassword,
|
|
||||||
}
|
|
||||||
|
|
||||||
export const postgresConfigCred = {
|
|
||||||
hostname: databaseHost,
|
|
||||||
port: parseInt(databasePort),
|
|
||||||
database: databaseNameCred,
|
|
||||||
user: databaseUser,
|
|
||||||
password: databasePassword,
|
|
||||||
};
|
|
@ -1,7 +1,14 @@
|
|||||||
import { Queue } from "bullmq";
|
import { Queue, ConnectionOptions } from "bullmq";
|
||||||
|
import { redis } from "@core/db/redis.ts";
|
||||||
|
|
||||||
export const LatestVideosQueue = new Queue("latestVideos");
|
export const LatestVideosQueue = new Queue("latestVideos", {
|
||||||
|
connection: redis as ConnectionOptions
|
||||||
|
});
|
||||||
|
|
||||||
export const ClassifyVideoQueue = new Queue("classifyVideo");
|
export const ClassifyVideoQueue = new Queue("classifyVideo", {
|
||||||
|
connection: redis as ConnectionOptions
|
||||||
|
});
|
||||||
|
|
||||||
export const SnapshotQueue = new Queue("snapshot");
|
export const SnapshotQueue = new Queue("snapshot", {
|
||||||
|
connection: redis as ConnectionOptions
|
||||||
|
});
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"name": "core",
|
"name": "crawler",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "bun --env-file=.env.test run vitest",
|
"test": "bun --env-file=.env.test run vitest",
|
||||||
"worker:main": "bun run ./src/worker.ts",
|
"worker:main": "bun run ./src/worker.ts",
|
||||||
|
Loading…
Reference in New Issue
Block a user