/** * AI Shortlist Service * * Generates ranked recommendations at end of evaluation rounds. * Follows patterns from ai-filtering.ts and ai-evaluation-summary.ts. * * GDPR Compliance: * - All project data is anonymized before AI processing * - No personal identifiers in prompts or responses */ import { getOpenAI, getConfiguredModel, buildCompletionParams } from '@/lib/openai' import { logAIUsage, extractTokenUsage } from '@/server/utils/ai-usage' import { classifyAIError, logAIError } from './ai-errors' import type { PrismaClient } from '@prisma/client' // ─── Types ────────────────────────────────────────────────────────────────── export type ShortlistResult = { success: boolean recommendations: ShortlistRecommendation[] errors?: string[] tokensUsed?: number } export type ShortlistRecommendation = { projectId: string rank: number score: number strengths: string[] concerns: string[] recommendation: string } // ─── Main Function ────────────────────────────────────────────────────────── /** * Generate an AI shortlist for projects in a round. * Only runs if EvaluationConfig.generateAiShortlist is true. */ export async function generateShortlist( params: { roundId: string competitionId: string category?: string topN?: number rubric?: string }, prisma: PrismaClient | any, ): Promise { const { roundId, competitionId, category, topN = 10, rubric } = params try { // Load projects with evaluations const where: Record = { assignments: { some: { roundId } }, } if (category) { where.competitionCategory = category } const projects = await prisma.project.findMany({ where, include: { assignments: { where: { roundId }, include: { evaluation: true, }, }, projectTags: { include: { tag: true } }, files: { select: { id: true, type: true } }, teamMembers: { select: { user: { select: { name: true } } } }, }, }) if (projects.length === 0) { return { success: true, recommendations: [], errors: ['No projects found for this round'], } } // Aggregate scores per project const projectSummaries = projects.map((project: any) => { const evaluations = project.assignments .map((a: any) => a.evaluation) .filter(Boolean) .filter((e: any) => e.status === 'SUBMITTED') const scores = evaluations.map((e: any) => e.globalScore ?? 0) const avgScore = scores.length > 0 ? scores.reduce((sum: number, s: number) => sum + s, 0) / scores.length : 0 const feedbacks = evaluations .map((e: any) => e.feedbackGeneral) .filter(Boolean) return { id: project.id, title: project.title, description: project.description, category: project.competitionCategory, tags: project.projectTags.map((pt: any) => pt.tag.name), avgScore, evaluationCount: evaluations.length, feedbackSamples: feedbacks.slice(0, 3), // Max 3 feedback samples } }) // Anonymize for AI const anonymized = projectSummaries.map((p: any, index: number) => ({ anonymousId: `PROJECT_${String(index + 1).padStart(3, '0')}`, ...p, // Strip identifying info title: undefined, id: undefined, })) // Build idMap for de-anonymization const idMap = new Map() projectSummaries.forEach((p: any, index: number) => { idMap.set(`PROJECT_${String(index + 1).padStart(3, '0')}`, p.id) }) // Build prompt const systemPrompt = `You are a senior jury advisor for the Monaco Ocean Protection Challenge. ## Your Role Analyze aggregated evaluation data to produce a ranked shortlist of top projects. ## Ranking Criteria (Weighted) - Evaluation Scores (40%): Average scores across all jury evaluations - Innovation & Impact (25%): Novelty of approach and potential environmental impact - Feasibility (20%): Likelihood of successful implementation - Alignment (15%): Fit with ocean protection mission and competition goals ## Output Format Return a JSON array: [ { "anonymousId": "PROJECT_001", "rank": 1, "score": 0-100, "strengths": ["strength 1", "strength 2"], "concerns": ["concern 1"], "recommendation": "1-2 sentence recommendation", "criterionBreakdown": { "evaluationScores": 38, "innovationImpact": 22, "feasibility": 18, "alignment": 14 } } ] ## Guidelines - Only include the requested number of top projects - Score should reflect weighted combination of all criteria - Be specific in strengths and concerns — avoid generic statements - Consider feedback themes and evaluator consensus - Higher evaluator consensus should boost confidence in ranking` const userPrompt = `Analyze these anonymized project evaluations and produce a ranked shortlist of the top ${topN} projects. ${rubric ? `Evaluation rubric:\n${rubric}\n\n` : ''}Projects: ${JSON.stringify(anonymized, null, 2)} Return a JSON array following the format specified in your instructions. Only include the top ${topN} projects. Rank by overall quality considering scores and feedback.` const openai = await getOpenAI() const model = await getConfiguredModel() if (!openai) { return { success: false, recommendations: [], errors: ['OpenAI client not configured'], } } const MAX_PARSE_RETRIES = 2 let parseAttempts = 0 let response = await openai.chat.completions.create( buildCompletionParams(model, { messages: [ { role: 'system', content: systemPrompt }, { role: 'user', content: userPrompt }, ], temperature: 0.1, jsonMode: true, }), ) let tokenUsage = extractTokenUsage(response) await logAIUsage({ action: 'FILTERING', model, promptTokens: tokenUsage.promptTokens, completionTokens: tokenUsage.completionTokens, totalTokens: tokenUsage.totalTokens, status: 'SUCCESS', }) // Parse response with retry logic let parsed: any[] while (true) { try { const content = response.choices[0]?.message?.content if (!content) { return { success: false, recommendations: [], errors: ['Empty AI response'], tokensUsed: tokenUsage.totalTokens, } } const json = JSON.parse(content) parsed = Array.isArray(json) ? json : json.rankings ?? json.projects ?? json.shortlist ?? [] break } catch (parseError) { if (parseError instanceof SyntaxError && parseAttempts < MAX_PARSE_RETRIES) { parseAttempts++ console.warn(`[AI Shortlist] JSON parse failed, retrying (${parseAttempts}/${MAX_PARSE_RETRIES})`) // Retry the API call with hint response = await openai.chat.completions.create( buildCompletionParams(model, { messages: [ { role: 'system', content: systemPrompt }, { role: 'user', content: userPrompt + '\n\nIMPORTANT: Please ensure valid JSON output.' }, ], temperature: 0.1, jsonMode: true, }), ) const retryUsage = extractTokenUsage(response) tokenUsage.totalTokens += retryUsage.totalTokens continue } return { success: false, recommendations: [], errors: ['Failed to parse AI response as JSON'], tokensUsed: tokenUsage.totalTokens, } } } // De-anonymize and build recommendations const recommendations: ShortlistRecommendation[] = parsed .filter((item: any) => item.anonymousId && idMap.has(item.anonymousId)) .map((item: any) => ({ projectId: idMap.get(item.anonymousId)!, rank: item.rank ?? 0, score: item.score ?? 0, strengths: item.strengths ?? [], concerns: item.concerns ?? [], recommendation: item.recommendation ?? '', })) .sort((a: ShortlistRecommendation, b: ShortlistRecommendation) => a.rank - b.rank) return { success: true, recommendations, tokensUsed: tokenUsage.totalTokens, } } catch (error) { const classification = classifyAIError(error) logAIError('ai-shortlist', 'generateShortlist', classification) console.error('[AIShortlist] generateShortlist failed:', error) return { success: false, recommendations: [], errors: [error instanceof Error ? error.message : 'AI shortlist generation failed'], } } }