diff --git a/src/server/services/ai-ranking.ts b/src/server/services/ai-ranking.ts new file mode 100644 index 0000000..a4255f3 --- /dev/null +++ b/src/server/services/ai-ranking.ts @@ -0,0 +1,428 @@ +/** + * AI Ranking Service + * + * Parses natural-language ranking criteria into structured rules and + * executes per-category project ranking using OpenAI. + * + * GDPR Compliance: + * - All project data is anonymized before AI processing (P001, P002, …) + * - No personal identifiers or real project IDs in prompts or responses + * + * Design decisions: + * - Per-category processing (STARTUP / BUSINESS_CONCEPT) — two parallel AI calls + * - Projects with zero submitted evaluations are excluded (not ranked last) + * - compositeScore = 50% normalised avgGlobalScore + 50% passRate + tiny tiebreak + */ + +import { getOpenAI, getConfiguredModel, buildCompletionParams } from '@/lib/openai' +import { logAIUsage, extractTokenUsage } from '@/server/utils/ai-usage' +import { classifyAIError, logAIError } from './ai-errors' +import { sanitizeUserInput } from '@/server/services/ai-prompt-guard' +import { TRPCError } from '@trpc/server' +import type { CompetitionCategory, PrismaClient } from '@prisma/client' + +// ─── Types ──────────────────────────────────────────────────────────────────── + +// Internal shape of a project before anonymization +interface ProjectForRanking { + id: string + competitionCategory: CompetitionCategory + avgGlobalScore: number | null // average of submitted Evaluation.globalScore + passRate: number // proportion of binaryDecision=true among SUBMITTED evaluations + evaluatorCount: number // count of SUBMITTED evaluations +} + +// Anonymized shape sent to OpenAI +interface AnonymizedProjectForRanking { + project_id: string // "P001", "P002", etc. — never real IDs + avg_score: number | null + pass_rate: number // 0–1 + evaluator_count: number + category: string +} + +// A single parsed rule returned by the criteria parser +export interface ParsedRankingRule { + step: number + type: 'filter' | 'sort' | 'limit' + description: string // Human-readable rule text + field: 'pass_rate' | 'avg_score' | 'evaluator_count' | null + operator: 'gte' | 'lte' | 'eq' | 'top_n' | null + value: number | null + dataAvailable: boolean // false = rule references unavailable data; UI should warn +} + +// A single project entry in the ranked output +export interface RankedProjectEntry { + projectId: string // Real project ID (de-anonymized) + rank: number // 1-indexed + compositeScore: number // 0–1 floating point + avgGlobalScore: number | null + passRate: number + evaluatorCount: number + aiRationale?: string // Optional: AI explanation for this project's rank +} + +// Full result for one category +export interface RankingResult { + category: CompetitionCategory + rankedProjects: RankedProjectEntry[] + parsedRules: ParsedRankingRule[] + totalEligible: number +} + +// ─── System Prompts ──────────────────────────────────────────────────────────── + +const CRITERIA_PARSING_SYSTEM_PROMPT = `You are a ranking criteria interpreter for an ocean conservation project competition (Monaco Ocean Protection Challenge). + +Admin will describe how they want projects ranked in natural language. Parse this into structured rules. + +Available data fields for ranking: +- avg_score: average jury evaluation score (1–10 scale, null if not scored) +- pass_rate: proportion of jury members who voted to advance the project (0–1) +- evaluator_count: number of jury members who submitted evaluations (tiebreak) + +Return JSON only: +{ + "rules": [ + { + "step": 1, + "type": "filter | sort | limit", + "description": "Human-readable description of this rule", + "field": "pass_rate | avg_score | evaluator_count | null", + "operator": "gte | lte | eq | top_n | null", + "value": , + "dataAvailable": true + } + ] +} + +Set dataAvailable=false if the rule requires data not in the available fields list above. Do NOT invent new fields. +Rules with dataAvailable=false will be shown as warnings to the admin — still include them. +Order rules so filters come first, sorts next, limits last.` + +const RANKING_SYSTEM_PROMPT = `You are a project ranking engine for an ocean conservation competition. + +You will receive a list of anonymized projects with numeric scores and a set of parsed ranking rules. +Apply the rules in order and return the final ranked list. + +Return JSON only: +{ + "ranked": [ + { + "project_id": "P001", + "rank": 1, + "rationale": "Brief explanation" + } + ] +} + +Rules: +- Apply filter rules first (remove projects that fail the filter) +- Apply sort rules next (order remaining projects) +- Apply limit rules last (keep only top N) +- Projects not in the ranked output are considered excluded (not ranked last) +- Use the project_id values exactly as given — do not change them` + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function computeCompositeScore( + avgGlobalScore: number | null, + passRate: number, + evaluatorCount: number, + maxEvaluatorCount: number, +): number { + const normalizedScore = avgGlobalScore != null ? (avgGlobalScore - 1) / 9 : 0.5 + const composite = normalizedScore * 0.5 + passRate * 0.5 + // Tiebreak: tiny bonus for more evaluators (won't change rank unless composite is equal) + const tiebreakBonus = maxEvaluatorCount > 0 + ? (evaluatorCount / maxEvaluatorCount) * 0.0001 + : 0 + return composite + tiebreakBonus +} + +function anonymizeProjectsForRanking( + projects: ProjectForRanking[], +): { anonymized: AnonymizedProjectForRanking[]; idMap: Map } { + const idMap = new Map() + const anonymized = projects.map((p, i) => { + const anonId = `P${String(i + 1).padStart(3, '0')}` + idMap.set(anonId, p.id) + return { + project_id: anonId, + avg_score: p.avgGlobalScore, + pass_rate: p.passRate, + evaluator_count: p.evaluatorCount, + category: p.competitionCategory, + } + }) + return { anonymized, idMap } +} + +/** + * Compute pass rate from Evaluation records. + * Handles both legacy binaryDecision boolean and future dedicated field. + * Falls back to binaryDecision if no future field exists. + */ +function computePassRate(evaluations: Array<{ binaryDecision: boolean | null }>): number { + if (evaluations.length === 0) return 0 + const passCount = evaluations.filter((e) => e.binaryDecision === true).length + return passCount / evaluations.length +} + +// ─── Exported Functions ─────────────────────────────────────────────────────── + +/** + * Parse natural-language ranking criteria into structured rules. + * Returns ParsedRankingRule[] for admin review (preview mode — RANK-02, RANK-03). + */ +export async function parseRankingCriteria( + criteriaText: string, + userId?: string, + entityId?: string, +): Promise { + const { sanitized: safeCriteria } = sanitizeUserInput(criteriaText) + + const openai = await getOpenAI() + if (!openai) { + throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' }) + } + + const model = await getConfiguredModel() + const params = buildCompletionParams(model, { + messages: [ + { role: 'system', content: CRITERIA_PARSING_SYSTEM_PROMPT }, + { role: 'user', content: safeCriteria }, + ], + jsonMode: true, + temperature: 0.1, + maxTokens: 1000, + }) + + let response: Awaited> + try { + response = await openai.chat.completions.create(params) + } catch (error) { + const classified = classifyAIError(error) + logAIError('Ranking', 'parseRankingCriteria', classified) + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message }) + } + + const usage = extractTokenUsage(response) + await logAIUsage({ + userId, + action: 'RANKING', + entityType: 'Round', + entityId, + model, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + totalTokens: usage.totalTokens, + itemsProcessed: 1, + status: 'SUCCESS', + }) + + const content = response.choices[0]?.message?.content + if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty response from AI' }) + + try { + const parsed = JSON.parse(content) as { rules: ParsedRankingRule[] } + return parsed.rules ?? [] + } catch { + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse AI response as JSON' }) + } +} + +/** + * Execute AI ranking for one category using pre-parsed rules. + * Returns RankingResult with ranked project list (RANK-05, RANK-06). + * + * projects: raw data queried from Prisma, already filtered to one category + * parsedRules: from parseRankingCriteria() + */ +export async function executeAIRanking( + parsedRules: ParsedRankingRule[], + projects: ProjectForRanking[], + category: CompetitionCategory, + userId?: string, + entityId?: string, +): Promise { + if (projects.length === 0) { + return { category, rankedProjects: [], parsedRules, totalEligible: 0 } + } + + const maxEvaluatorCount = Math.max(...projects.map((p) => p.evaluatorCount)) + const { anonymized, idMap } = anonymizeProjectsForRanking(projects) + + const openai = await getOpenAI() + if (!openai) { + throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' }) + } + + const model = await getConfiguredModel() + + const userPrompt = JSON.stringify({ + rules: parsedRules.filter((r) => r.dataAvailable), + projects: anonymized, + }) + + const params = buildCompletionParams(model, { + messages: [ + { role: 'system', content: RANKING_SYSTEM_PROMPT }, + { role: 'user', content: userPrompt }, + ], + jsonMode: true, + temperature: 0, + maxTokens: 2000, + }) + + let response: Awaited> + try { + response = await openai.chat.completions.create(params) + } catch (error) { + const classified = classifyAIError(error) + logAIError('Ranking', 'executeAIRanking', classified) + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message }) + } + + const usage = extractTokenUsage(response) + await logAIUsage({ + userId, + action: 'RANKING', + entityType: 'Round', + entityId, + model, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + totalTokens: usage.totalTokens, + itemsProcessed: projects.length, + status: 'SUCCESS', + }) + + const content = response.choices[0]?.message?.content + if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty ranking response from AI' }) + + let aiRanked: Array<{ project_id: string; rank: number; rationale?: string }> + try { + const parsed = JSON.parse(content) as { ranked: typeof aiRanked } + aiRanked = parsed.ranked ?? [] + } catch { + throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse ranking response as JSON' }) + } + + // Build a lookup by anonymousId for project data + const projectByAnonId = new Map( + anonymized.map((a) => [a.project_id, projects.find((p) => p.id === idMap.get(a.project_id))!]) + ) + + const rankedProjects: RankedProjectEntry[] = aiRanked + .filter((entry) => idMap.has(entry.project_id)) + .map((entry) => { + const realId = idMap.get(entry.project_id)! + const proj = projectByAnonId.get(entry.project_id)! + return { + projectId: realId, + rank: entry.rank, + compositeScore: computeCompositeScore( + proj.avgGlobalScore, + proj.passRate, + proj.evaluatorCount, + maxEvaluatorCount, + ), + avgGlobalScore: proj.avgGlobalScore, + passRate: proj.passRate, + evaluatorCount: proj.evaluatorCount, + aiRationale: entry.rationale, + } + }) + .sort((a, b) => a.rank - b.rank) + + return { + category, + rankedProjects, + parsedRules, + totalEligible: projects.length, + } +} + +/** + * Quick-rank: parse criteria and execute ranking in one step. + * Returns results for all categories (RANK-04). + * The prisma parameter is used to fetch project evaluation data. + */ +export async function quickRank( + criteriaText: string, + roundId: string, + prisma: PrismaClient, + userId?: string, +): Promise<{ startup: RankingResult; concept: RankingResult; parsedRules: ParsedRankingRule[] }> { + const parsedRules = await parseRankingCriteria(criteriaText, userId, roundId) + const [startup, concept] = await Promise.all([ + fetchAndRankCategory('STARTUP', parsedRules, roundId, prisma, userId), + fetchAndRankCategory('BUSINESS_CONCEPT', parsedRules, roundId, prisma, userId), + ]) + return { startup, concept, parsedRules } +} + +/** + * Internal helper: fetch eligible projects for one category and execute ranking. + * Excluded: withdrawn projects and projects with zero submitted evaluations (locked decision). + * + * Exported so the tRPC router can call it separately when executing pre-parsed rules. + */ +export async function fetchAndRankCategory( + category: CompetitionCategory, + parsedRules: ParsedRankingRule[], + roundId: string, + prisma: PrismaClient, + userId?: string, +): Promise { + // Query submitted evaluations grouped by projectId for this category + const assignments = await prisma.assignment.findMany({ + where: { + roundId, + isRequired: true, + project: { + competitionCategory: category, + // Exclude withdrawn projects + projectRoundStates: { + none: { roundId, state: 'WITHDRAWN' }, + }, + }, + evaluation: { + status: 'SUBMITTED', // Only count completed evaluations + }, + }, + include: { + evaluation: { + select: { globalScore: true, binaryDecision: true }, + }, + project: { + select: { id: true, competitionCategory: true }, + }, + }, + }) + + // Group by projectId + const byProject = new Map>() + for (const a of assignments) { + if (!a.evaluation) continue + const list = byProject.get(a.project.id) ?? [] + list.push({ globalScore: a.evaluation.globalScore, binaryDecision: a.evaluation.binaryDecision }) + byProject.set(a.project.id, list) + } + + // Build ProjectForRanking, excluding projects with zero submitted evaluations + const projects: ProjectForRanking[] = [] + for (const [projectId, evals] of byProject.entries()) { + if (evals.length === 0) continue // Exclude: no submitted evaluations + const avgGlobalScore = evals.some((e) => e.globalScore != null) + ? evals.filter((e) => e.globalScore != null).reduce((sum, e) => sum + e.globalScore!, 0) / + evals.filter((e) => e.globalScore != null).length + : null + const passRate = computePassRate(evals) + projects.push({ id: projectId, competitionCategory: category, avgGlobalScore, passRate, evaluatorCount: evals.length }) + } + + return executeAIRanking(parsedRules, projects, category, userId, roundId) +}