feat(01-02): create ai-ranking.ts service with criteria parsing and ranking

- Add parseRankingCriteria() — parses natural-language criteria via OpenAI JSON mode - Add executeAIRanking() — anonymizes projects (P001…), calls OpenAI, de-anonymizes results - Add quickRank() — one-shot helper that parses + ranks both categories in parallel - Add fetchAndRankCategory() — fetches eligible projects from Prisma and calls executeAIRanking - compositeScore: 50% normalised avgGlobalScore + 50% passRate + tiny tiebreak bonus - Projects with zero SUBMITTED evaluations are excluded (not ranked last) - All project IDs anonymized before OpenAI — no PII in prompts - Follows ai-filtering.ts pattern: getOpenAI, logAIUsage with action RANKING, classifyAIError
2026-02-27 00:48:09 +01:00
parent 7193abd87b
commit aa383f53f8
1 changed files with 428 additions and 0 deletions
--- a/src/server/services/ai-ranking.ts
+++ b/src/server/services/ai-ranking.ts
@@ -0,0 +1,428 @@
+/**
+ * AI Ranking Service
+ *
+ * Parses natural-language ranking criteria into structured rules and
+ * executes per-category project ranking using OpenAI.
+ *
+ * GDPR Compliance:
+ * - All project data is anonymized before AI processing (P001, P002, …)
+ * - No personal identifiers or real project IDs in prompts or responses
+ *
+ * Design decisions:
+ * - Per-category processing (STARTUP / BUSINESS_CONCEPT) — two parallel AI calls
+ * - Projects with zero submitted evaluations are excluded (not ranked last)
+ * - compositeScore = 50% normalised avgGlobalScore + 50% passRate + tiny tiebreak
+ */
+
+import { getOpenAI, getConfiguredModel, buildCompletionParams } from '@/lib/openai'
+import { logAIUsage, extractTokenUsage } from '@/server/utils/ai-usage'
+import { classifyAIError, logAIError } from './ai-errors'
+import { sanitizeUserInput } from '@/server/services/ai-prompt-guard'
+import { TRPCError } from '@trpc/server'
+import type { CompetitionCategory, PrismaClient } from '@prisma/client'
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+// Internal shape of a project before anonymization
+interface ProjectForRanking {
+  id: string
+  competitionCategory: CompetitionCategory
+  avgGlobalScore: number | null      // average of submitted Evaluation.globalScore
+  passRate: number                   // proportion of binaryDecision=true among SUBMITTED evaluations
+  evaluatorCount: number             // count of SUBMITTED evaluations
+}
+
+// Anonymized shape sent to OpenAI
+interface AnonymizedProjectForRanking {
+  project_id: string  // "P001", "P002", etc. — never real IDs
+  avg_score: number | null
+  pass_rate: number   // 0–1
+  evaluator_count: number
+  category: string
+}
+
+// A single parsed rule returned by the criteria parser
+export interface ParsedRankingRule {
+  step: number
+  type: 'filter' | 'sort' | 'limit'
+  description: string               // Human-readable rule text
+  field: 'pass_rate' | 'avg_score' | 'evaluator_count' | null
+  operator: 'gte' | 'lte' | 'eq' | 'top_n' | null
+  value: number | null
+  dataAvailable: boolean            // false = rule references unavailable data; UI should warn
+}
+
+// A single project entry in the ranked output
+export interface RankedProjectEntry {
+  projectId: string          // Real project ID (de-anonymized)
+  rank: number               // 1-indexed
+  compositeScore: number     // 0–1 floating point
+  avgGlobalScore: number | null
+  passRate: number
+  evaluatorCount: number
+  aiRationale?: string       // Optional: AI explanation for this project's rank
+}
+
+// Full result for one category
+export interface RankingResult {
+  category: CompetitionCategory
+  rankedProjects: RankedProjectEntry[]
+  parsedRules: ParsedRankingRule[]
+  totalEligible: number
+}
+
+// ─── System Prompts ────────────────────────────────────────────────────────────
+
+const CRITERIA_PARSING_SYSTEM_PROMPT = `You are a ranking criteria interpreter for an ocean conservation project competition (Monaco Ocean Protection Challenge).
+
+Admin will describe how they want projects ranked in natural language. Parse this into structured rules.
+
+Available data fields for ranking:
+- avg_score: average jury evaluation score (1–10 scale, null if not scored)
+- pass_rate: proportion of jury members who voted to advance the project (0–1)
+- evaluator_count: number of jury members who submitted evaluations (tiebreak)
+
+Return JSON only:
+{
+  "rules": [
+    {
+      "step": 1,
+      "type": "filter | sort | limit",
+      "description": "Human-readable description of this rule",
+      "field": "pass_rate | avg_score | evaluator_count | null",
+      "operator": "gte | lte | eq | top_n | null",
+      "value": <number or null>,
+      "dataAvailable": true
+    }
+  ]
+}
+
+Set dataAvailable=false if the rule requires data not in the available fields list above. Do NOT invent new fields.
+Rules with dataAvailable=false will be shown as warnings to the admin — still include them.
+Order rules so filters come first, sorts next, limits last.`
+
+const RANKING_SYSTEM_PROMPT = `You are a project ranking engine for an ocean conservation competition.
+
+You will receive a list of anonymized projects with numeric scores and a set of parsed ranking rules.
+Apply the rules in order and return the final ranked list.
+
+Return JSON only:
+{
+  "ranked": [
+    {
+      "project_id": "P001",
+      "rank": 1,
+      "rationale": "Brief explanation"
+    }
+  ]
+}
+
+Rules:
+- Apply filter rules first (remove projects that fail the filter)
+- Apply sort rules next (order remaining projects)
+- Apply limit rules last (keep only top N)
+- Projects not in the ranked output are considered excluded (not ranked last)
+- Use the project_id values exactly as given — do not change them`
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+function computeCompositeScore(
+  avgGlobalScore: number | null,
+  passRate: number,
+  evaluatorCount: number,
+  maxEvaluatorCount: number,
+): number {
+  const normalizedScore = avgGlobalScore != null ? (avgGlobalScore - 1) / 9 : 0.5
+  const composite = normalizedScore * 0.5 + passRate * 0.5
+  // Tiebreak: tiny bonus for more evaluators (won't change rank unless composite is equal)
+  const tiebreakBonus = maxEvaluatorCount > 0
+    ? (evaluatorCount / maxEvaluatorCount) * 0.0001
+    : 0
+  return composite + tiebreakBonus
+}
+
+function anonymizeProjectsForRanking(
+  projects: ProjectForRanking[],
+): { anonymized: AnonymizedProjectForRanking[]; idMap: Map<string, string> } {
+  const idMap = new Map<string, string>()
+  const anonymized = projects.map((p, i) => {
+    const anonId = `P${String(i + 1).padStart(3, '0')}`
+    idMap.set(anonId, p.id)
+    return {
+      project_id: anonId,
+      avg_score: p.avgGlobalScore,
+      pass_rate: p.passRate,
+      evaluator_count: p.evaluatorCount,
+      category: p.competitionCategory,
+    }
+  })
+  return { anonymized, idMap }
+}
+
+/**
+ * Compute pass rate from Evaluation records.
+ * Handles both legacy binaryDecision boolean and future dedicated field.
+ * Falls back to binaryDecision if no future field exists.
+ */
+function computePassRate(evaluations: Array<{ binaryDecision: boolean | null }>): number {
+  if (evaluations.length === 0) return 0
+  const passCount = evaluations.filter((e) => e.binaryDecision === true).length
+  return passCount / evaluations.length
+}
+
+// ─── Exported Functions ───────────────────────────────────────────────────────
+
+/**
+ * Parse natural-language ranking criteria into structured rules.
+ * Returns ParsedRankingRule[] for admin review (preview mode — RANK-02, RANK-03).
+ */
+export async function parseRankingCriteria(
+  criteriaText: string,
+  userId?: string,
+  entityId?: string,
+): Promise<ParsedRankingRule[]> {
+  const { sanitized: safeCriteria } = sanitizeUserInput(criteriaText)
+
+  const openai = await getOpenAI()
+  if (!openai) {
+    throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' })
+  }
+
+  const model = await getConfiguredModel()
+  const params = buildCompletionParams(model, {
+    messages: [
+      { role: 'system', content: CRITERIA_PARSING_SYSTEM_PROMPT },
+      { role: 'user', content: safeCriteria },
+    ],
+    jsonMode: true,
+    temperature: 0.1,
+    maxTokens: 1000,
+  })
+
+  let response: Awaited<ReturnType<typeof openai.chat.completions.create>>
+  try {
+    response = await openai.chat.completions.create(params)
+  } catch (error) {
+    const classified = classifyAIError(error)
+    logAIError('Ranking', 'parseRankingCriteria', classified)
+    throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message })
+  }
+
+  const usage = extractTokenUsage(response)
+  await logAIUsage({
+    userId,
+    action: 'RANKING',
+    entityType: 'Round',
+    entityId,
+    model,
+    promptTokens: usage.promptTokens,
+    completionTokens: usage.completionTokens,
+    totalTokens: usage.totalTokens,
+    itemsProcessed: 1,
+    status: 'SUCCESS',
+  })
+
+  const content = response.choices[0]?.message?.content
+  if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty response from AI' })
+
+  try {
+    const parsed = JSON.parse(content) as { rules: ParsedRankingRule[] }
+    return parsed.rules ?? []
+  } catch {
+    throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse AI response as JSON' })
+  }
+}
+
+/**
+ * Execute AI ranking for one category using pre-parsed rules.
+ * Returns RankingResult with ranked project list (RANK-05, RANK-06).
+ *
+ * projects: raw data queried from Prisma, already filtered to one category
+ * parsedRules: from parseRankingCriteria()
+ */
+export async function executeAIRanking(
+  parsedRules: ParsedRankingRule[],
+  projects: ProjectForRanking[],
+  category: CompetitionCategory,
+  userId?: string,
+  entityId?: string,
+): Promise<RankingResult> {
+  if (projects.length === 0) {
+    return { category, rankedProjects: [], parsedRules, totalEligible: 0 }
+  }
+
+  const maxEvaluatorCount = Math.max(...projects.map((p) => p.evaluatorCount))
+  const { anonymized, idMap } = anonymizeProjectsForRanking(projects)
+
+  const openai = await getOpenAI()
+  if (!openai) {
+    throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' })
+  }
+
+  const model = await getConfiguredModel()
+
+  const userPrompt = JSON.stringify({
+    rules: parsedRules.filter((r) => r.dataAvailable),
+    projects: anonymized,
+  })
+
+  const params = buildCompletionParams(model, {
+    messages: [
+      { role: 'system', content: RANKING_SYSTEM_PROMPT },
+      { role: 'user', content: userPrompt },
+    ],
+    jsonMode: true,
+    temperature: 0,
+    maxTokens: 2000,
+  })
+
+  let response: Awaited<ReturnType<typeof openai.chat.completions.create>>
+  try {
+    response = await openai.chat.completions.create(params)
+  } catch (error) {
+    const classified = classifyAIError(error)
+    logAIError('Ranking', 'executeAIRanking', classified)
+    throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message })
+  }
+
+  const usage = extractTokenUsage(response)
+  await logAIUsage({
+    userId,
+    action: 'RANKING',
+    entityType: 'Round',
+    entityId,
+    model,
+    promptTokens: usage.promptTokens,
+    completionTokens: usage.completionTokens,
+    totalTokens: usage.totalTokens,
+    itemsProcessed: projects.length,
+    status: 'SUCCESS',
+  })
+
+  const content = response.choices[0]?.message?.content
+  if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty ranking response from AI' })
+
+  let aiRanked: Array<{ project_id: string; rank: number; rationale?: string }>
+  try {
+    const parsed = JSON.parse(content) as { ranked: typeof aiRanked }
+    aiRanked = parsed.ranked ?? []
+  } catch {
+    throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse ranking response as JSON' })
+  }
+
+  // Build a lookup by anonymousId for project data
+  const projectByAnonId = new Map(
+    anonymized.map((a) => [a.project_id, projects.find((p) => p.id === idMap.get(a.project_id))!])
+  )
+
+  const rankedProjects: RankedProjectEntry[] = aiRanked
+    .filter((entry) => idMap.has(entry.project_id))
+    .map((entry) => {
+      const realId = idMap.get(entry.project_id)!
+      const proj = projectByAnonId.get(entry.project_id)!
+      return {
+        projectId: realId,
+        rank: entry.rank,
+        compositeScore: computeCompositeScore(
+          proj.avgGlobalScore,
+          proj.passRate,
+          proj.evaluatorCount,
+          maxEvaluatorCount,
+        ),
+        avgGlobalScore: proj.avgGlobalScore,
+        passRate: proj.passRate,
+        evaluatorCount: proj.evaluatorCount,
+        aiRationale: entry.rationale,
+      }
+    })
+    .sort((a, b) => a.rank - b.rank)
+
+  return {
+    category,
+    rankedProjects,
+    parsedRules,
+    totalEligible: projects.length,
+  }
+}
+
+/**
+ * Quick-rank: parse criteria and execute ranking in one step.
+ * Returns results for all categories (RANK-04).
+ * The prisma parameter is used to fetch project evaluation data.
+ */
+export async function quickRank(
+  criteriaText: string,
+  roundId: string,
+  prisma: PrismaClient,
+  userId?: string,
+): Promise<{ startup: RankingResult; concept: RankingResult; parsedRules: ParsedRankingRule[] }> {
+  const parsedRules = await parseRankingCriteria(criteriaText, userId, roundId)
+  const [startup, concept] = await Promise.all([
+    fetchAndRankCategory('STARTUP', parsedRules, roundId, prisma, userId),
+    fetchAndRankCategory('BUSINESS_CONCEPT', parsedRules, roundId, prisma, userId),
+  ])
+  return { startup, concept, parsedRules }
+}
+
+/**
+ * Internal helper: fetch eligible projects for one category and execute ranking.
+ * Excluded: withdrawn projects and projects with zero submitted evaluations (locked decision).
+ *
+ * Exported so the tRPC router can call it separately when executing pre-parsed rules.
+ */
+export async function fetchAndRankCategory(
+  category: CompetitionCategory,
+  parsedRules: ParsedRankingRule[],
+  roundId: string,
+  prisma: PrismaClient,
+  userId?: string,
+): Promise<RankingResult> {
+  // Query submitted evaluations grouped by projectId for this category
+  const assignments = await prisma.assignment.findMany({
+    where: {
+      roundId,
+      isRequired: true,
+      project: {
+        competitionCategory: category,
+        // Exclude withdrawn projects
+        projectRoundStates: {
+          none: { roundId, state: 'WITHDRAWN' },
+        },
+      },
+      evaluation: {
+        status: 'SUBMITTED',  // Only count completed evaluations
+      },
+    },
+    include: {
+      evaluation: {
+        select: { globalScore: true, binaryDecision: true },
+      },
+      project: {
+        select: { id: true, competitionCategory: true },
+      },
+    },
+  })
+
+  // Group by projectId
+  const byProject = new Map<string, Array<{ globalScore: number | null; binaryDecision: boolean | null }>>()
+  for (const a of assignments) {
+    if (!a.evaluation) continue
+    const list = byProject.get(a.project.id) ?? []
+    list.push({ globalScore: a.evaluation.globalScore, binaryDecision: a.evaluation.binaryDecision })
+    byProject.set(a.project.id, list)
+  }
+
+  // Build ProjectForRanking, excluding projects with zero submitted evaluations
+  const projects: ProjectForRanking[] = []
+  for (const [projectId, evals] of byProject.entries()) {
+    if (evals.length === 0) continue  // Exclude: no submitted evaluations
+    const avgGlobalScore = evals.some((e) => e.globalScore != null)
+      ? evals.filter((e) => e.globalScore != null).reduce((sum, e) => sum + e.globalScore!, 0) /
+        evals.filter((e) => e.globalScore != null).length
+      : null
+    const passRate = computePassRate(evals)
+    projects.push({ id: projectId, competitionCategory: category, avgGlobalScore, passRate, evaluatorCount: evals.length })
+  }
+
+  return executeAIRanking(parsedRules, projects, category, userId, roundId)
+}