diff --git a/Dockerfile b/Dockerfile index a81e426..c8ee3cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ RUN npm install --production=false FROM base AS builder COPY --from=deps /app/node_modules ./node_modules COPY . . +RUN npx prisma generate --schema=packages/database/prisma/schema.prisma RUN npx turbo build FROM base AS web @@ -24,6 +25,7 @@ RUN addgroup --system --gid 1001 nodejs && \ COPY --from=builder --chown=nextjs:nodejs /app/apps/web/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/apps/web/.next/static ./apps/web/.next/static COPY --from=builder --chown=nextjs:nodejs /app/apps/web/public ./apps/web/public +COPY --from=builder --chown=nextjs:nodejs /app/packages/database/prisma ./packages/database/prisma USER nextjs EXPOSE 3000 ENV PORT=3000 HOSTNAME="0.0.0.0" @@ -38,6 +40,10 @@ COPY --from=builder /app/packages/shared/dist ./packages/shared/dist COPY --from=builder /app/packages/parser/dist ./packages/parser/dist COPY --from=builder /app/packages/llm/dist ./packages/llm/dist COPY --from=builder /app/packages/diagrams/dist ./packages/diagrams/dist +COPY --from=builder /app/packages/database/package.json ./packages/database/ +COPY --from=builder /app/node_modules/.prisma ./node_modules/.prisma +COPY --from=builder /app/node_modules/@prisma ./node_modules/@prisma +COPY --from=builder /app/packages/database/prisma ./packages/database/prisma COPY --from=builder /app/package.json ./ COPY --from=builder /app/apps/worker/package.json ./apps/worker/ COPY --from=builder /app/packages/shared/package.json ./packages/shared/ diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index b075a2d..3cc3f6d 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -1,6 +1,6 @@ /** @type {import('next').NextConfig} */ const config = { - transpilePackages: ["@codeboard/shared"], + transpilePackages: ["@codeboard/shared", "@codeboard/database"], output: "standalone", }; diff --git a/apps/web/package.json b/apps/web/package.json index b743abc..45832ed 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@codeboard/shared": "*", + "@codeboard/database": "*", "@tailwindcss/typography": "^0.5.19", "bullmq": "^5.34.0", "ioredis": "^5.4.0", diff --git a/apps/web/src/app/api/docs/[id]/route.ts b/apps/web/src/app/api/docs/[id]/route.ts index 8fd387f..1fc0584 100644 --- a/apps/web/src/app/api/docs/[id]/route.ts +++ b/apps/web/src/app/api/docs/[id]/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from "next/server"; import { getRedis } from "@/lib/redis"; +import { prisma } from "@codeboard/database"; export async function GET( _request: Request, @@ -8,13 +9,34 @@ export async function GET( const { id } = await params; const redis = getRedis(); - const result = await redis.get(`codeboard:result:${id}`); - if (!result) { + let result = await redis.get(`codeboard:result:${id}`); + + if (result) { + return NextResponse.json(JSON.parse(result)); + } + + const generation = await prisma.generation.findFirst({ + where: { id } + }); + + if (!generation || !generation.result) { return NextResponse.json( { error: "Documentation not found" }, { status: 404 } ); } - return NextResponse.json(JSON.parse(result)); + const docs = generation.result as any; + docs.id = id; + docs.repoUrl = generation.repoUrl; + docs.repoName = generation.repoName; + + await redis.set( + `codeboard:result:${id}`, + JSON.stringify(docs), + "EX", + 86400 + ); + + return NextResponse.json(docs); } diff --git a/apps/web/src/app/api/generate/route.ts b/apps/web/src/app/api/generate/route.ts index 83b0e58..cc7831e 100644 --- a/apps/web/src/app/api/generate/route.ts +++ b/apps/web/src/app/api/generate/route.ts @@ -28,7 +28,7 @@ export async function POST(request: Request) { const queue = getQueue(); await queue.add("generate", { repoUrl, generationId }, { jobId: generationId, - removeOnComplete: true, + removeOnComplete: { age: 3600 }, removeOnFail: false, }); diff --git a/apps/worker/package.json b/apps/worker/package.json index 55996c7..630a0e3 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -13,6 +13,7 @@ "@codeboard/parser": "*", "@codeboard/llm": "*", "@codeboard/diagrams": "*", + "@codeboard/database": "*", "bullmq": "^5.34.0", "ioredis": "^5.4.0", "simple-git": "^3.27.0" diff --git a/apps/worker/src/processor.ts b/apps/worker/src/processor.ts index e31e65b..73b8e75 100644 --- a/apps/worker/src/processor.ts +++ b/apps/worker/src/processor.ts @@ -1,5 +1,6 @@ import type { Job } from "bullmq"; import IORedis from "ioredis"; +import { prisma } from "@codeboard/database"; import { cloneRepository } from "./jobs/clone.js"; import { parseRepository } from "./jobs/parse.js"; import { generateDocs } from "./jobs/generate.js"; @@ -38,6 +39,33 @@ export async function processGenerationJob( try { await updateProgress(generationId, "CLONING", 10, "Cloning repository..."); const cloneResult = await cloneRepository(repoUrl); + const commitHash = cloneResult.metadata.lastCommit; + + const existingGeneration = await prisma.generation.findUnique({ + where: { + repoUrl_commitHash: { + repoUrl, + commitHash + } + } + }); + + if (existingGeneration && existingGeneration.result) { + const docs = existingGeneration.result as any; + docs.id = generationId; + docs.repoUrl = repoUrl; + docs.repoName = existingGeneration.repoName; + + await redis.set( + `codeboard:result:${generationId}`, + JSON.stringify(docs), + "EX", + 86400 + ); + + await updateProgress(generationId, "COMPLETED", 100, "Using cached documentation!"); + return { generationId, duration: 0, repoName: existingGeneration.repoName, cached: true }; + } await updateProgress( generationId, @@ -72,9 +100,22 @@ export async function processGenerationJob( 86400 ); + await prisma.generation.create({ + data: { + id: generationId, + repoUrl, + repoName: cloneResult.metadata.name, + commitHash, + status: "COMPLETED", + progress: 100, + result: docs as any, + duration + } + }); + await updateProgress(generationId, "COMPLETED", 100, "Done!"); - return { generationId, duration, repoName: cloneResult.metadata.name }; + return { generationId, duration, repoName: cloneResult.metadata.name, cached: false }; } catch (err) { const message = err instanceof Error ? err.message : "Unknown error"; await updateProgress(generationId, "FAILED", 0, message); diff --git a/docker-compose.yml b/docker-compose.yml index 29951b1..0a07e50 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,9 +7,14 @@ services: - "4100:3000" environment: - REDIS_URL=redis://redis:6379 + - DATABASE_URL=postgresql://codeboard:codeboard@postgres:5432/codeboard depends_on: redis: condition: service_started + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully restart: always worker: @@ -18,6 +23,7 @@ services: target: worker environment: - REDIS_URL=redis://redis:6379 + - DATABASE_URL=postgresql://codeboard:codeboard@postgres:5432/codeboard - OPENAI_API_KEY=${OPENAI_API_KEY:-} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} - LLM_MODEL=${LLM_MODEL:-kimi-k2-turbo-preview} @@ -25,8 +31,39 @@ services: depends_on: redis: condition: service_started + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully restart: always + postgres: + image: postgres:16-alpine + environment: + - POSTGRES_USER=codeboard + - POSTGRES_PASSWORD=codeboard + - POSTGRES_DB=codeboard + volumes: + - codeboard_postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U codeboard"] + interval: 5s + timeout: 5s + retries: 5 + restart: always + + migrate: + build: + context: . + target: builder + command: npx prisma migrate deploy --schema=packages/database/prisma/schema.prisma + environment: + - DATABASE_URL=postgresql://codeboard:codeboard@postgres:5432/codeboard + depends_on: + postgres: + condition: service_healthy + restart: "no" + redis: image: redis:7-alpine volumes: @@ -34,4 +71,5 @@ services: restart: always volumes: + codeboard_postgres_data: codeboard_redis_data: diff --git a/packages/database/prisma/migrations/0001_init/migration.sql b/packages/database/prisma/migrations/0001_init/migration.sql new file mode 100644 index 0000000..aee6876 --- /dev/null +++ b/packages/database/prisma/migrations/0001_init/migration.sql @@ -0,0 +1,49 @@ +-- CreateEnum +CREATE TYPE "Status" AS ENUM ('QUEUED', 'CLONING', 'PARSING', 'GENERATING', 'RENDERING', 'COMPLETED', 'FAILED'); + +-- CreateTable +CREATE TABLE "Generation" ( + "id" TEXT NOT NULL, + "repoUrl" TEXT NOT NULL, + "repoName" TEXT NOT NULL, + "commitHash" TEXT NOT NULL, + "status" "Status" NOT NULL DEFAULT 'QUEUED', + "progress" INTEGER NOT NULL DEFAULT 0, + "result" JSONB, + "error" TEXT, + "costUsd" DOUBLE PRECISION, + "duration" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "userId" TEXT, + "viewCount" INTEGER NOT NULL DEFAULT 0, + + CONSTRAINT "Generation_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "User" ( + "id" TEXT NOT NULL, + "githubId" TEXT NOT NULL, + "login" TEXT NOT NULL, + "email" TEXT, + "avatarUrl" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "User_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "Generation_repoUrl_idx" ON "Generation"("repoUrl"); + +-- CreateIndex +CREATE INDEX "Generation_status_idx" ON "Generation"("status"); + +-- CreateIndex +CREATE UNIQUE INDEX "Generation_repoUrl_commitHash_key" ON "Generation"("repoUrl", "commitHash"); + +-- CreateIndex +CREATE UNIQUE INDEX "User_githubId_key" ON "User"("githubId"); + +-- AddForeignKey +ALTER TABLE "Generation" ADD CONSTRAINT "Generation_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/database/prisma/migrations/migration_lock.toml b/packages/database/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..99e4f20 --- /dev/null +++ b/packages/database/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "postgresql" diff --git a/turbo.json b/turbo.json index f0dc45e..48f223f 100644 --- a/turbo.json +++ b/turbo.json @@ -18,6 +18,9 @@ "db:generate": { "cache": false }, + "db:migrate": { + "cache": false + }, "db:push": { "cache": false }