diff --git a/.idea/data_source_mapping.xml b/.idea/data_source_mapping.xml index e7530ed..42a0f80 100644 --- a/.idea/data_source_mapping.xml +++ b/.idea/data_source_mapping.xml @@ -12,5 +12,7 @@ + + \ No newline at end of file diff --git a/.idea/db-forest-config.xml b/.idea/db-forest-config.xml index b821497..122530f 100644 --- a/.idea/db-forest-config.xml +++ b/.idea/db-forest-config.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/bun.lock b/bun.lock index 9a97ada..9e35672 100644 --- a/bun.lock +++ b/bun.lock @@ -5,10 +5,13 @@ "": { "name": "cvsa", "dependencies": { + "@t3-oss/env-core": "^0.13.10", "arg": "^5.0.2", + "dayjs": "^1.11.19", "dotenv": "^17.2.3", "drizzle-orm": "^0.44.7", "postgres": "^3.4.7", + "zod": "^4.3.5", }, "devDependencies": { "@biomejs/biome": "2.3.8", @@ -957,6 +960,8 @@ "@speed-highlight/core": ["@speed-highlight/core@1.2.14", "", {}, "sha512-G4ewlBNhUtlLvrJTb88d2mdy2KRijzs4UhnlrOSRT4bmjh/IqNElZa3zkrZ+TC47TwtlDWzVLFADljF1Ijp5hA=="], + "@t3-oss/env-core": ["@t3-oss/env-core@0.13.10", "", { "peerDependencies": { "arktype": "^2.1.0", "typescript": ">=5.0.0", "valibot": "^1.0.0-beta.7 || ^1.0.0", "zod": "^3.24.0 || ^4.0.0" }, "optionalPeers": ["arktype", "typescript", "valibot", "zod"] }, "sha512-NNFfdlJ+HmPHkLi2HKy7nwuat9SIYOxei9K10lO2YlcSObDILY7mHZNSHsieIM3A0/5OOzw/P/b+yLvPdaG52g=="], + "@tailwindcss/node": ["@tailwindcss/node@4.1.18", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "enhanced-resolve": "^5.18.3", "jiti": "^2.6.1", "lightningcss": "1.30.2", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.1.18" } }, "sha512-DoR7U1P7iYhw16qJ49fgXUlry1t4CpXeErJHnQ44JgTSKMaZUdf17cfn5mHchfJ4KRBZRFA/Coo+MUF5+gOaCQ=="], "@tailwindcss/oxide": ["@tailwindcss/oxide@4.1.18", "", { "optionalDependencies": { "@tailwindcss/oxide-android-arm64": "4.1.18", "@tailwindcss/oxide-darwin-arm64": "4.1.18", "@tailwindcss/oxide-darwin-x64": "4.1.18", "@tailwindcss/oxide-freebsd-x64": "4.1.18", "@tailwindcss/oxide-linux-arm-gnueabihf": "4.1.18", "@tailwindcss/oxide-linux-arm64-gnu": "4.1.18", "@tailwindcss/oxide-linux-arm64-musl": "4.1.18", "@tailwindcss/oxide-linux-x64-gnu": "4.1.18", "@tailwindcss/oxide-linux-x64-musl": "4.1.18", "@tailwindcss/oxide-wasm32-wasi": "4.1.18", "@tailwindcss/oxide-win32-arm64-msvc": "4.1.18", "@tailwindcss/oxide-win32-x64-msvc": "4.1.18" } }, "sha512-EgCR5tTS5bUSKQgzeMClT6iCY3ToqE1y+ZB0AKldj809QXk1Y+3jB0upOYZrn9aGIzPtUsP7sX4QQ4XtjBB95A=="], @@ -1431,6 +1436,8 @@ "date-fns-jalali": ["date-fns-jalali@4.1.0-0", "", {}, "sha512-hTIP/z+t+qKwBDcmmsnmjWTduxCg+5KfdqWQvb2X/8C9+knYY6epN/pfxdDuyVlSVeFz0sM5eEfwIUQ70U4ckg=="], + "dayjs": ["dayjs@1.11.19", "", {}, "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "decamelize": ["decamelize@4.0.0", "", {}, "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ=="], diff --git a/package.json b/package.json index dd692dc..cdace1c 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,13 @@ } }, "dependencies": { + "@t3-oss/env-core": "^0.13.10", "arg": "^5.0.2", + "dayjs": "^1.11.19", "dotenv": "^17.2.3", "drizzle-orm": "^0.44.7", - "postgres": "^3.4.7" + "postgres": "^3.4.7", + "zod": "^4.3.5" }, "devDependencies": { "@biomejs/biome": "2.3.8", diff --git a/packages/core/db/pgConfigNew.ts b/packages/core/db/pgConfigNew.ts index 796998f..9964153 100644 --- a/packages/core/db/pgConfigNew.ts +++ b/packages/core/db/pgConfigNew.ts @@ -1,6 +1,6 @@ -const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT", "DB_NAME_CRED"]; +const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT"]; -const getEnvVar = (key: string) => { +const getEnvVar = (key: string): string => { return process.env[key] || import.meta.env[key]; }; @@ -10,25 +10,16 @@ if (unsetVars.length > 0) { throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`); } -const databaseHost = getEnvVar("DB_HOST")!; +const databaseHost = getEnvVar("DB_HOST"); const databaseName = getEnvVar("DB_NAME"); -const databaseNameCred = getEnvVar("DB_NAME_CRED")!; -const databaseUser = getEnvVar("DB_USER")!; -const databasePassword = getEnvVar("DB_PASSWORD")!; -const databasePort = getEnvVar("DB_PORT")!; +const databaseUser = getEnvVar("DB_USER"); +const databasePassword = getEnvVar("DB_PASSWORD"); +const databasePort = getEnvVar("DB_PORT"); export const postgresConfig = { database: databaseName, host: databaseHost, password: databasePassword, - port: parseInt(databasePort), + port: parseInt(databasePort, 10), username: databaseUser, }; - -export const postgresConfigCred = { - database: databaseNameCred, - hostname: databaseHost, - password: databasePassword, - port: parseInt(databasePort), - user: databaseUser, -}; diff --git a/src/backup.ts b/src/backup.ts new file mode 100644 index 0000000..03c7f56 --- /dev/null +++ b/src/backup.ts @@ -0,0 +1,116 @@ +import { postgresConfig } from "@core/db/pgConfigNew"; +import logger from "@core/log"; +import { $, S3Client } from "bun"; +import dayjs from "dayjs"; +import { env } from "./env"; + +const AK = env.OSS_ACCESS_KEY_ID; +const SK = env.OSS_ACCESS_KEY_SECRET; +const ENDPOINT = env.BACKUP_S3_ENDPOINT; +const REGION = env.BACKUP_S3_REGION; +const BUCKET = env.BACKUP_S3_BUCKET; +const DIR = env.BACKUP_DIR; + +const CONFIG = { + localBackupDir: DIR, + retentionDaily: 3, + s3: { + bucket: BUCKET, + endpoint: ENDPOINT, + region: REGION, + }, +}; + +const { username, password, host, port, database } = postgresConfig; +const dbUri = `postgresql://${username}:${encodeURIComponent(password)}@${host}:${port}/${database}`; + +const s3 = new S3Client({ + accessKeyId: AK, + bucket: CONFIG.s3.bucket, + endpoint: CONFIG.s3.endpoint, + region: CONFIG.s3.region, + secretAccessKey: SK, + virtualHostedStyle: true, +}); + +const getDayStr = (): string => { + return dayjs().format("YYYY-MM-DD"); +}; + +const getMonthStr = (): string => { + return dayjs().format("YYYY-MM"); +}; + +async function runBackup() { + const dayStr = getDayStr(); + const monthStr = getMonthStr(); + const fileName = `cvsa_${dayStr}.dump`; + const filePath = `${CONFIG.localBackupDir}/${fileName}.dump`; + const localDumpfile = Bun.file(filePath); + + logger.log(`Starting backup...`); + + if (!(await localDumpfile.exists())) { + logger.log(`Creating dump ${localDumpfile.name}...`); + const cmd = $`pg_dump -d ${dbUri} -Fc -n public > ${filePath}`; + + await cmd; + } + + const monthlyBackupFile = s3.file(`dump/monthly/${monthStr}`); + + if (!(await monthlyBackupFile.exists())) { + logger.log(`Uploading ${filePath} to ${monthlyBackupFile.name}`); + await monthlyBackupFile.write(localDumpfile); + } + + const dailyBackupFile = s3.file(`dump/daily/${dayStr}`); + + if (!(await dailyBackupFile.exists())) { + logger.log(`Uploading ${filePath} to ${dailyBackupFile.name}`); + await dailyBackupFile.write(localDumpfile); + } +} + +async function rotateS3Backups() { + const dailyBackups = await s3.list({ + maxKeys: 1000, + prefix: "dump/daily/", + }); + if (!dailyBackups.contents) { + logger.log("No daily backups found"); + return; + } + logger.log(`Found ${dailyBackups.contents.length} daily backups`); + for (const content of dailyBackups.contents) { + const key = content.key; + if (!key) { + continue; + } + const dateStr = key.split("/").at(-1); + const date = dayjs(dateStr, "YYYY-MM-DD"); + if (date.isBefore(dayjs().subtract(CONFIG.retentionDaily, "day"))) { + logger.log(`Deleting daily backup ${key}`); + await s3.delete(key); + } + } + + if (dailyBackups.isTruncated) { + logger.log("Still more daily backups"); + await rotateS3Backups(); + } + logger.log("Daily backups rotated"); +} + +async function main() { + try { + await runBackup(); + await rotateS3Backups(); + } catch (err) { + logger.error(err); + process.exit(1); + } +} + +await main(); +process.exit(); diff --git a/src/env.ts b/src/env.ts new file mode 100644 index 0000000..af5d5e4 --- /dev/null +++ b/src/env.ts @@ -0,0 +1,14 @@ +import { createEnv } from "@t3-oss/env-core"; +import { z } from "zod"; + +export const env = createEnv({ + runtimeEnv: Bun.env, + server: { + BACKUP_DIR: z.string(), + BACKUP_S3_BUCKET: z.string(), + BACKUP_S3_ENDPOINT: z.string(), + BACKUP_S3_REGION: z.string(), + OSS_ACCESS_KEY_ID: z.string(), + OSS_ACCESS_KEY_SECRET: z.string(), + }, +});