Feature/appbuilder-grader

local.env.dist

@@ -22,6 +22,7 @@ BUILD_ENGINE_SECRETS_BUCKET=SOME_ORG_PREFIX-SOME_APP_ENV-aps-secrets
 # Built artifacts will be stored here
 BUILD_ENGINE_ARTIFACTS_BUCKET=SOME_ORG_PREFIX-SOME_APP_ENV-aps-artifacts
 BUILD_ENGINE_ARTIFACTS_BUCKET_REGION=us-east-1
+BUILD_ENGINE_GRADING_LAMBDA_FUNCTION_NAME=SOME_GRADING_LAMBDA_FUNCTION
 
 # Projects are moved to here by SAB
 BUILD_ENGINE_PROJECTS_BUCKET=SOME_ORG_PREFIX-SOME_APP_ENV-aps-projects

package.json

@@ -67,6 +67,7 @@
 		"@aws-sdk/client-codebuild": "^3.907.0",
 		"@aws-sdk/client-ecr": "^3.981.0",
 		"@aws-sdk/client-iam": "^3.911.0",
+		"@aws-sdk/client-lambda": "^3.907.0",
 		"@aws-sdk/client-s3": "^3.907.0",
 		"@aws-sdk/client-sts": "^3.907.0",
 		"@grpc/grpc-js": "^1.14.3",

src/lib/prisma/migrations/03_grading_result/migration.sql

@@ -0,0 +1,19 @@
+-- CreateTable
+CREATE TABLE "public"."gradingResult" (
+    "uuid" UUID NOT NULL,
+    "project_id" INTEGER NOT NULL,
+    "status" VARCHAR(255),
+    "result" VARCHAR(2000),
+    "publisher_id" VARCHAR(255) NOT NULL,
+    "lambda_request_id" VARCHAR(255),
+    "created" TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP,
+    "updated" TIMESTAMP(6),
+
+    CONSTRAINT "gradingResult_pkey" PRIMARY KEY ("uuid")
+);
+
+-- CreateIndex
+CREATE INDEX "idx_grading_result_project_id" ON "public"."gradingResult"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "public"."gradingResult" ADD CONSTRAINT "fk_grading_result_project_id" FOREIGN KEY ("project_id") REFERENCES "public"."project"("id") ON DELETE NO ACTION ON UPDATE NO ACTION;

src/lib/prisma/schema.prisma

@@ -4,7 +4,7 @@ generator client {
 
 datasource db {
   provider = "postgresql"
-  url = env("DATABASE_URL")
+  url      = env("DATABASE_URL")
 }
 
 model build {
@@ -62,21 +62,22 @@ model job {
 }
 
 model project {
-  id             Int       @id @default(autoincrement())
-  status         String?   @db.VarChar(255)
-  result         String?   @db.VarChar(255)
-  error          String?   @db.VarChar(2083)
-  url            String?   @db.VarChar(1024)
-  user_id        String?   @db.VarChar(255) // ISSUE #77: remove this?
-  group_id       String?   @db.VarChar(255) // ISSUE #77: remove this?
-  app_id         String?   @db.VarChar(255)
-  project_name   String?   @db.VarChar(255)
-  language_code  String?   @db.VarChar(255)
-  publishing_key String?   @db.VarChar(1024) // ISSUE #77: remove this?
-  created        DateTime? @default(now()) @db.Timestamp(6)
-  updated        DateTime? @updatedAt @db.Timestamp(6)
+  id             Int             @id @default(autoincrement())
+  status         String?         @db.VarChar(255)
+  result         String?         @db.VarChar(255)
+  error          String?         @db.VarChar(2083)
+  url            String?         @db.VarChar(1024)
+  user_id        String?         @db.VarChar(255) // ISSUE #77: remove this?
+  group_id       String?         @db.VarChar(255) // ISSUE #77: remove this?
+  app_id         String?         @db.VarChar(255)
+  project_name   String?         @db.VarChar(255)
+  language_code  String?         @db.VarChar(255)
+  publishing_key String?         @db.VarChar(1024) // ISSUE #77: remove this?
+  created        DateTime?       @default(now()) @db.Timestamp(6)
+  updated        DateTime?       @updatedAt @db.Timestamp(6)
   client_id      Int?
-  client         client?   @relation(fields: [client_id], references: [id], onDelete: NoAction, onUpdate: NoAction, map: "fk_project_client_id")
+  gradingResult  gradingResult[]
+  client         client?         @relation(fields: [client_id], references: [id], onDelete: NoAction, onUpdate: NoAction, map: "fk_project_client_id")
 
   @@index([client_id], map: "idx_project_client_id")
 }
@@ -112,3 +113,17 @@ model appVersion {
   created   DateTime  @default(now()) @db.Timestamp(6)
   updated   DateTime? @updatedAt @db.Timestamp(6)
 }
+
+model gradingResult {
+  uuid              String    @id @default(uuid()) @db.Uuid
+  project_id        Int
+  status            String?   @db.VarChar(255)
+  result            String?   @db.VarChar(2000)
+  publisher_id      String    @db.VarChar(255)
+  lambda_request_id String?   @db.VarChar(255)
+  created           DateTime? @default(now()) @db.Timestamp(6)
+  updated           DateTime? @updatedAt @db.Timestamp(6)
+  project           project   @relation(fields: [project_id], references: [id], onDelete: NoAction, onUpdate: NoAction, map: "fk_grading_result_project_id")
+
+  @@index([project_id], map: "idx_grading_result_project_id")
+}

src/lib/server/aws/lambda.ts

@@ -0,0 +1,65 @@
+import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda';
+import { SpanStatusCode, trace } from '@opentelemetry/api';
+import { AWSVars } from './vars';
+
+const tracer = trace.getTracer('Lambda');
+
+export class Lambda {
+  private client;
+
+  public constructor() {
+    this.client = new LambdaClient({ region: AWSVars.artifactsRegion() });
+  }
+
+  public async invokeJson<TPayload extends Record<string, unknown>>(
+    functionName: string,
+    payload: TPayload
+  ) {
+    return tracer.startActiveSpan('Lambda - Invoke', async (span) => {
+      span.setAttributes({
+        'lambda.function-name': functionName,
+        'lambda.payload': JSON.stringify(payload)
+      });
+      try {
+        const startTime = Date.now();
+        const result = await this.client.send(
+          new InvokeCommand({
+            FunctionName: functionName,
+            InvocationType: 'RequestResponse',
+            Payload: Buffer.from(JSON.stringify(payload))
+          })
+        );
+        span.setAttributes({
+          'lambda.status-code': result.StatusCode,
+          'lambda.executed-version': result.ExecutedVersion ?? '',
+          'lambda.function-error': result.FunctionError ?? '',
+          'lambda.executionTimeMs': Date.now() - startTime
+        });
+
+        const body = result.Payload ? Buffer.from(result.Payload).toString('utf8') : '';
+        const parsed = body ? this.parsePayload(body) : null;
+        return {
+          requestId: result.$metadata.requestId ?? null,
+          payload: parsed
+        };
+      } catch (e) {
+        span.recordException(e as Error);
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: (e as Error).message
+        });
+        throw e;
+      } finally {
+        span.end();
+      }
+    });
+  }
+
+  private parsePayload(body: string) {
+    try {
+      return JSON.parse(body) as Record<string, unknown>;
+    } catch {
+      throw new Error(`Lambda returned invalid JSON: ${body}`);
+    }
+  }
+}

src/lib/server/aws/s3.ts

@@ -2,6 +2,7 @@ import {
   CopyObjectCommand,
   DeleteObjectsCommand,
   GetObjectCommand,
+  HeadObjectCommand,
   ListObjectsV2Command,
   NoSuchKey,
   PutObjectCommand,
@@ -237,6 +238,25 @@ export class S3 {
     );
     handleArtifact(artifacts_provider, fileS3Key, fileContents);
   }
+
+  public async objectExists(key: string) {
+    const bucket = AWSVars.artifacts();
+    try {
+      await this.s3Client.send(
+        new HeadObjectCommand({
+          Bucket: bucket,
+          Key: key
+        })
+      );
+      return true;
+    } catch (e) {
+      if (e instanceof S3ServiceException && e.$metadata.httpStatusCode === 404) {
+        return false;
+      }
+      throw e;
+    }
+  }
+
   public async removeCodeBuildFolder(artifacts_provider: ProviderForPrefix) {
     const s3Folder = getBasePrefixUrl(artifacts_provider, 'codebuild-output') + '/';
     const s3Bucket = AWSVars.artifacts();

src/lib/server/aws/vars.ts

@@ -42,6 +42,10 @@ export class AWSVars {
   public static scriptsPath() {
     return `s3://${AWSVars.projects()}/default`;
   }
+
+  public static gradingLambdaFunctionName() {
+    return env.BUILD_ENGINE_GRADING_LAMBDA_FUNCTION_NAME;
+  }
   /**
    *  Get the project name which is the prd or stg plus build_app or publish_app
    *

src/lib/server/bullmq/BullMQ.ts

@@ -7,6 +7,7 @@ export const allWorkers = building
       new Workers.Builds(),
       new Workers.S3(),
       new Workers.Releases(),
+      new Workers.Grading(),
       new Workers.Polling(),
       new Workers.SystemStartup(),
       new Workers.SystemRecurring()

src/lib/server/bullmq/BullWorker.ts

@@ -150,6 +150,18 @@ export class Releases<J extends BullMQ.PublishJob> extends BullWorker<J> {
   }
 }
 
+export class Grading<J extends BullMQ.GradingJob> extends BullWorker<J> {
+  constructor() {
+    super(BullMQ.QueueName.Grading);
+  }
+  async run(job: Job<J>) {
+    switch (job.data.type) {
+      case BullMQ.JobType.Grading_Generate:
+        return Executor.Grading.generate(job as Job<BullMQ.Grading.Generate>);
+    }
+  }
+}
+
 export class Polling<J extends BullMQ.PollJob> extends BullWorker<J> {
   constructor() {
     super(BullMQ.QueueName.Polling);

src/lib/server/bullmq/queues.ts

@@ -1,7 +1,15 @@
 import { Queue } from 'bullmq';
 import { BullMQOtel } from 'bullmq-otel';
 import { Redis } from 'ioredis';
-import type { BuildJob, PollJob, PublishJob, RecurringJob, S3Job, StartupJob } from './types';
+import type {
+  BuildJob,
+  GradingJob,
+  PollJob,
+  PublishJob,
+  RecurringJob,
+  S3Job,
+  StartupJob
+} from './types';
 import { QueueName } from './types';
 import { env } from '$env/dynamic/private';
 import OTEL from '$lib/otel';
@@ -123,6 +131,8 @@ function createQueues() {
   const S3 = new Queue<S3Job>(QueueName.S3, getQueueConfig());
   /** Queue for Product Publishing  */
   const Releases = new Queue<PublishJob>(QueueName.Releases, getQueueConfig());
+  /** Queue for Grading report generation */
+  const Grading = new Queue<GradingJob>(QueueName.Grading, getQueueConfig());
   /** Queue for jobs that poll BuildEngine, such as checking the status of a build */
   const Polling = new Queue<PollJob>(QueueName.Polling, getQueueConfig());
   /** Queue for jobs that run on startup, such as creating the CodeBuild project */
@@ -133,6 +143,7 @@ function createQueues() {
     Builds,
     S3,
     Releases,
+    Grading,
     Polling,
     SystemStartup,
     SystemRecurring

src/lib/server/bullmq/types.ts

@@ -20,6 +20,7 @@ export enum QueueName {
   Builds = 'Builds',
   S3 = 'S3',
   Releases = 'Releases',
+  Grading = 'Grading',
   Polling = 'Polling',
   System_Startup = 'System (Startup)',
   System_Recurring = 'System (Recurring)'
@@ -35,6 +36,8 @@ export enum JobType {
   // Publishing Jobs
   Release_Product = 'Release Product',
   Release_Cancel = 'Cancel Release',
+  // Grading Jobs
+  Grading_Generate = 'Generate Grading Report',
   // S3 Jobs
   S3_CopyArtifacts = 'Copy Artifacts to S3',
   S3_CopyError = 'Copy Errors to S3',
@@ -85,6 +88,13 @@ export namespace Release {
   }
 }
 
+export namespace Grading {
+  export interface Generate {
+    type: JobType.Grading_Generate;
+    gradingResultUUID: string;
+  }
+}
+
 export namespace S3 {
   export interface CopyArtifacts {
     type: JobType.S3_CopyArtifacts;
@@ -112,6 +122,7 @@ export type Job = JobTypeMap[keyof JobTypeMap];
 export type BuildJob = JobTypeMap[JobType.Build_Product | JobType.Build_Cancel];
 export type S3Job = JobTypeMap[JobType.S3_CopyArtifacts | JobType.S3_CopyError];
 export type PublishJob = JobTypeMap[JobType.Release_Product | JobType.Release_Cancel];
+export type GradingJob = JobTypeMap[JobType.Grading_Generate];
 export type PollJob = JobTypeMap[JobType.Poll_Build | JobType.Poll_Release];
 export type StartupJob = JobTypeMap[
   | JobType.System_CreateCodeBuildProject
@@ -125,6 +136,7 @@ export type JobTypeMap = {
   [JobType.Poll_Release]: Polling.Release;
   [JobType.Release_Product]: Release.Product;
   [JobType.Release_Cancel]: Release.Cancel;
+  [JobType.Grading_Generate]: Grading.Generate;
   [JobType.S3_CopyArtifacts]: S3.CopyArtifacts;
   [JobType.S3_CopyError]: S3.CopyErrors;
   [JobType.System_CreateCodeBuildProject]: System.CreateCodeBuildProject;