/** * AI Ranking Service * * Parses natural-language ranking criteria into structured rules and * executes per-category project ranking using OpenAI. * * GDPR Compliance: * - All project data is anonymized before AI processing (P001, P002, …) * - No personal identifiers or real project IDs in prompts or responses * * Design decisions: * - Per-category processing (STARTUP / BUSINESS_CONCEPT) — two parallel AI calls * - Projects with zero submitted evaluations are excluded (not ranked last) * - compositeScore = 50% normalised avgGlobalScore + 50% passRate + tiny tiebreak */ import { getOpenAI, getConfiguredModel, buildCompletionParams } from '@/lib/openai' import { logAIUsage, extractTokenUsage } from '@/server/utils/ai-usage' import { classifyAIError, logAIError } from './ai-errors' import { sanitizeUserInput } from '@/server/services/ai-prompt-guard' import { TRPCError } from '@trpc/server' import type { CompetitionCategory, PrismaClient } from '@prisma/client' // ─── Types ──────────────────────────────────────────────────────────────────── // Internal shape of a project before anonymization interface ProjectForRanking { id: string competitionCategory: CompetitionCategory avgGlobalScore: number | null // average of submitted Evaluation.globalScore passRate: number // proportion of binaryDecision=true among SUBMITTED evaluations evaluatorCount: number // count of SUBMITTED evaluations } // Anonymized shape sent to OpenAI interface AnonymizedProjectForRanking { project_id: string // "P001", "P002", etc. — never real IDs avg_score: number | null pass_rate: number // 0–1 evaluator_count: number category: string } // A single parsed rule returned by the criteria parser export interface ParsedRankingRule { step: number type: 'filter' | 'sort' | 'limit' description: string // Human-readable rule text field: 'pass_rate' | 'avg_score' | 'evaluator_count' | null operator: 'gte' | 'lte' | 'eq' | 'top_n' | null value: number | null dataAvailable: boolean // false = rule references unavailable data; UI should warn } // A single project entry in the ranked output export interface RankedProjectEntry { projectId: string // Real project ID (de-anonymized) rank: number // 1-indexed compositeScore: number // 0–1 floating point avgGlobalScore: number | null passRate: number evaluatorCount: number aiRationale?: string // Optional: AI explanation for this project's rank } // Full result for one category export interface RankingResult { category: CompetitionCategory rankedProjects: RankedProjectEntry[] parsedRules: ParsedRankingRule[] totalEligible: number } // ─── System Prompts ──────────────────────────────────────────────────────────── const CRITERIA_PARSING_SYSTEM_PROMPT = `You are a ranking criteria interpreter for an ocean conservation project competition (Monaco Ocean Protection Challenge). Admin will describe how they want projects ranked in natural language. Parse this into structured rules. Available data fields for ranking: - avg_score: average jury evaluation score (1–10 scale, null if not scored) - pass_rate: proportion of jury members who voted to advance the project (0–1) - evaluator_count: number of jury members who submitted evaluations (tiebreak) Return JSON only: { "rules": [ { "step": 1, "type": "filter | sort | limit", "description": "Human-readable description of this rule", "field": "pass_rate | avg_score | evaluator_count | null", "operator": "gte | lte | eq | top_n | null", "value": , "dataAvailable": true } ] } Set dataAvailable=false if the rule requires data not in the available fields list above. Do NOT invent new fields. Rules with dataAvailable=false will be shown as warnings to the admin — still include them. Order rules so filters come first, sorts next, limits last.` const RANKING_SYSTEM_PROMPT = `You are a project ranking engine for an ocean conservation competition. You will receive a list of anonymized projects with numeric scores and a set of parsed ranking rules. Apply the rules in order and return the final ranked list. Return JSON only: { "ranked": [ { "project_id": "P001", "rank": 1, "rationale": "Brief explanation" } ] } Rules: - Apply filter rules first (remove projects that fail the filter) - Apply sort rules next (order remaining projects) - Apply limit rules last (keep only top N) - Projects not in the ranked output are considered excluded (not ranked last) - Use the project_id values exactly as given — do not change them` // ─── Helpers ────────────────────────────────────────────────────────────────── function computeCompositeScore( avgGlobalScore: number | null, passRate: number, evaluatorCount: number, maxEvaluatorCount: number, ): number { const normalizedScore = avgGlobalScore != null ? (avgGlobalScore - 1) / 9 : 0.5 const composite = normalizedScore * 0.5 + passRate * 0.5 // Tiebreak: tiny bonus for more evaluators (won't change rank unless composite is equal) const tiebreakBonus = maxEvaluatorCount > 0 ? (evaluatorCount / maxEvaluatorCount) * 0.0001 : 0 return composite + tiebreakBonus } function anonymizeProjectsForRanking( projects: ProjectForRanking[], ): { anonymized: AnonymizedProjectForRanking[]; idMap: Map } { const idMap = new Map() const anonymized = projects.map((p, i) => { const anonId = `P${String(i + 1).padStart(3, '0')}` idMap.set(anonId, p.id) return { project_id: anonId, avg_score: p.avgGlobalScore, pass_rate: p.passRate, evaluator_count: p.evaluatorCount, category: p.competitionCategory, } }) return { anonymized, idMap } } /** * Compute pass rate from Evaluation records. * Handles both legacy binaryDecision boolean and future dedicated field. * Falls back to binaryDecision if no future field exists. */ function computePassRate(evaluations: Array<{ binaryDecision: boolean | null }>): number { if (evaluations.length === 0) return 0 const passCount = evaluations.filter((e) => e.binaryDecision === true).length return passCount / evaluations.length } // ─── Exported Functions ─────────────────────────────────────────────────────── /** * Parse natural-language ranking criteria into structured rules. * Returns ParsedRankingRule[] for admin review (preview mode — RANK-02, RANK-03). */ export async function parseRankingCriteria( criteriaText: string, userId?: string, entityId?: string, ): Promise { const { sanitized: safeCriteria } = sanitizeUserInput(criteriaText) const openai = await getOpenAI() if (!openai) { throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' }) } const model = await getConfiguredModel() const params = buildCompletionParams(model, { messages: [ { role: 'system', content: CRITERIA_PARSING_SYSTEM_PROMPT }, { role: 'user', content: safeCriteria }, ], jsonMode: true, temperature: 0.1, maxTokens: 1000, }) let response: Awaited> try { response = await openai.chat.completions.create(params) } catch (error) { const classified = classifyAIError(error) logAIError('Ranking', 'parseRankingCriteria', classified) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message }) } const usage = extractTokenUsage(response) await logAIUsage({ userId, action: 'RANKING', entityType: 'Round', entityId, model, promptTokens: usage.promptTokens, completionTokens: usage.completionTokens, totalTokens: usage.totalTokens, itemsProcessed: 1, status: 'SUCCESS', }) const content = response.choices[0]?.message?.content if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty response from AI' }) try { const parsed = JSON.parse(content) as { rules: ParsedRankingRule[] } return parsed.rules ?? [] } catch { throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse AI response as JSON' }) } } /** * Execute AI ranking for one category using pre-parsed rules. * Returns RankingResult with ranked project list (RANK-05, RANK-06). * * projects: raw data queried from Prisma, already filtered to one category * parsedRules: from parseRankingCriteria() */ export async function executeAIRanking( parsedRules: ParsedRankingRule[], projects: ProjectForRanking[], category: CompetitionCategory, userId?: string, entityId?: string, ): Promise { if (projects.length === 0) { return { category, rankedProjects: [], parsedRules, totalEligible: 0 } } const maxEvaluatorCount = Math.max(...projects.map((p) => p.evaluatorCount)) const { anonymized, idMap } = anonymizeProjectsForRanking(projects) const openai = await getOpenAI() if (!openai) { throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' }) } const model = await getConfiguredModel() const userPrompt = JSON.stringify({ rules: parsedRules.filter((r) => r.dataAvailable), projects: anonymized, }) const params = buildCompletionParams(model, { messages: [ { role: 'system', content: RANKING_SYSTEM_PROMPT }, { role: 'user', content: userPrompt }, ], jsonMode: true, temperature: 0, maxTokens: 2000, }) let response: Awaited> try { response = await openai.chat.completions.create(params) } catch (error) { const classified = classifyAIError(error) logAIError('Ranking', 'executeAIRanking', classified) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message }) } const usage = extractTokenUsage(response) await logAIUsage({ userId, action: 'RANKING', entityType: 'Round', entityId, model, promptTokens: usage.promptTokens, completionTokens: usage.completionTokens, totalTokens: usage.totalTokens, itemsProcessed: projects.length, status: 'SUCCESS', }) const content = response.choices[0]?.message?.content if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty ranking response from AI' }) let aiRanked: Array<{ project_id: string; rank: number; rationale?: string }> try { const parsed = JSON.parse(content) as { ranked: typeof aiRanked } aiRanked = parsed.ranked ?? [] } catch { throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse ranking response as JSON' }) } // Build a lookup by anonymousId for project data const projectByAnonId = new Map( anonymized.map((a) => [a.project_id, projects.find((p) => p.id === idMap.get(a.project_id))!]) ) const rankedProjects: RankedProjectEntry[] = aiRanked .filter((entry) => idMap.has(entry.project_id)) .map((entry) => { const realId = idMap.get(entry.project_id)! const proj = projectByAnonId.get(entry.project_id)! return { projectId: realId, rank: entry.rank, compositeScore: computeCompositeScore( proj.avgGlobalScore, proj.passRate, proj.evaluatorCount, maxEvaluatorCount, ), avgGlobalScore: proj.avgGlobalScore, passRate: proj.passRate, evaluatorCount: proj.evaluatorCount, aiRationale: entry.rationale, } }) .sort((a, b) => a.rank - b.rank) return { category, rankedProjects, parsedRules, totalEligible: projects.length, } } /** * Quick-rank: parse criteria and execute ranking in one step. * Returns results for all categories (RANK-04). * The prisma parameter is used to fetch project evaluation data. */ export async function quickRank( criteriaText: string, roundId: string, prisma: PrismaClient, userId?: string, ): Promise<{ startup: RankingResult; concept: RankingResult; parsedRules: ParsedRankingRule[] }> { const parsedRules = await parseRankingCriteria(criteriaText, userId, roundId) const [startup, concept] = await Promise.all([ fetchAndRankCategory('STARTUP', parsedRules, roundId, prisma, userId), fetchAndRankCategory('BUSINESS_CONCEPT', parsedRules, roundId, prisma, userId), ]) return { startup, concept, parsedRules } } /** * Internal helper: fetch eligible projects for one category and execute ranking. * Excluded: withdrawn projects and projects with zero submitted evaluations (locked decision). * * Exported so the tRPC router can call it separately when executing pre-parsed rules. */ export async function fetchAndRankCategory( category: CompetitionCategory, parsedRules: ParsedRankingRule[], roundId: string, prisma: PrismaClient, userId?: string, ): Promise { // Query submitted evaluations grouped by projectId for this category const assignments = await prisma.assignment.findMany({ where: { roundId, isRequired: true, project: { competitionCategory: category, // Exclude withdrawn projects projectRoundStates: { none: { roundId, state: 'WITHDRAWN' }, }, }, evaluation: { status: 'SUBMITTED', // Only count completed evaluations }, }, include: { evaluation: { select: { globalScore: true, binaryDecision: true }, }, project: { select: { id: true, competitionCategory: true }, }, }, }) // Group by projectId const byProject = new Map>() for (const a of assignments) { if (!a.evaluation) continue const list = byProject.get(a.project.id) ?? [] list.push({ globalScore: a.evaluation.globalScore, binaryDecision: a.evaluation.binaryDecision }) byProject.set(a.project.id, list) } // Build ProjectForRanking, excluding projects with zero submitted evaluations const projects: ProjectForRanking[] = [] for (const [projectId, evals] of byProject.entries()) { if (evals.length === 0) continue // Exclude: no submitted evaluations const avgGlobalScore = evals.some((e) => e.globalScore != null) ? evals.filter((e) => e.globalScore != null).reduce((sum, e) => sum + e.globalScore!, 0) / evals.filter((e) => e.globalScore != null).length : null const passRate = computePassRate(evals) projects.push({ id: projectId, competitionCategory: category, avgGlobalScore, passRate, evaluatorCount: evals.length }) } return executeAIRanking(parsedRules, projects, category, userId, roundId) }