feat(01-02): create ai-ranking.ts service with criteria parsing and ranking
- Add parseRankingCriteria() — parses natural-language criteria via OpenAI JSON mode - Add executeAIRanking() — anonymizes projects (P001…), calls OpenAI, de-anonymizes results - Add quickRank() — one-shot helper that parses + ranks both categories in parallel - Add fetchAndRankCategory() — fetches eligible projects from Prisma and calls executeAIRanking - compositeScore: 50% normalised avgGlobalScore + 50% passRate + tiny tiebreak bonus - Projects with zero SUBMITTED evaluations are excluded (not ranked last) - All project IDs anonymized before OpenAI — no PII in prompts - Follows ai-filtering.ts pattern: getOpenAI, logAIUsage with action RANKING, classifyAIError
This commit is contained in:
428
src/server/services/ai-ranking.ts
Normal file
428
src/server/services/ai-ranking.ts
Normal file
@@ -0,0 +1,428 @@
|
||||
/**
|
||||
* AI Ranking Service
|
||||
*
|
||||
* Parses natural-language ranking criteria into structured rules and
|
||||
* executes per-category project ranking using OpenAI.
|
||||
*
|
||||
* GDPR Compliance:
|
||||
* - All project data is anonymized before AI processing (P001, P002, …)
|
||||
* - No personal identifiers or real project IDs in prompts or responses
|
||||
*
|
||||
* Design decisions:
|
||||
* - Per-category processing (STARTUP / BUSINESS_CONCEPT) — two parallel AI calls
|
||||
* - Projects with zero submitted evaluations are excluded (not ranked last)
|
||||
* - compositeScore = 50% normalised avgGlobalScore + 50% passRate + tiny tiebreak
|
||||
*/
|
||||
|
||||
import { getOpenAI, getConfiguredModel, buildCompletionParams } from '@/lib/openai'
|
||||
import { logAIUsage, extractTokenUsage } from '@/server/utils/ai-usage'
|
||||
import { classifyAIError, logAIError } from './ai-errors'
|
||||
import { sanitizeUserInput } from '@/server/services/ai-prompt-guard'
|
||||
import { TRPCError } from '@trpc/server'
|
||||
import type { CompetitionCategory, PrismaClient } from '@prisma/client'
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Internal shape of a project before anonymization
|
||||
interface ProjectForRanking {
|
||||
id: string
|
||||
competitionCategory: CompetitionCategory
|
||||
avgGlobalScore: number | null // average of submitted Evaluation.globalScore
|
||||
passRate: number // proportion of binaryDecision=true among SUBMITTED evaluations
|
||||
evaluatorCount: number // count of SUBMITTED evaluations
|
||||
}
|
||||
|
||||
// Anonymized shape sent to OpenAI
|
||||
interface AnonymizedProjectForRanking {
|
||||
project_id: string // "P001", "P002", etc. — never real IDs
|
||||
avg_score: number | null
|
||||
pass_rate: number // 0–1
|
||||
evaluator_count: number
|
||||
category: string
|
||||
}
|
||||
|
||||
// A single parsed rule returned by the criteria parser
|
||||
export interface ParsedRankingRule {
|
||||
step: number
|
||||
type: 'filter' | 'sort' | 'limit'
|
||||
description: string // Human-readable rule text
|
||||
field: 'pass_rate' | 'avg_score' | 'evaluator_count' | null
|
||||
operator: 'gte' | 'lte' | 'eq' | 'top_n' | null
|
||||
value: number | null
|
||||
dataAvailable: boolean // false = rule references unavailable data; UI should warn
|
||||
}
|
||||
|
||||
// A single project entry in the ranked output
|
||||
export interface RankedProjectEntry {
|
||||
projectId: string // Real project ID (de-anonymized)
|
||||
rank: number // 1-indexed
|
||||
compositeScore: number // 0–1 floating point
|
||||
avgGlobalScore: number | null
|
||||
passRate: number
|
||||
evaluatorCount: number
|
||||
aiRationale?: string // Optional: AI explanation for this project's rank
|
||||
}
|
||||
|
||||
// Full result for one category
|
||||
export interface RankingResult {
|
||||
category: CompetitionCategory
|
||||
rankedProjects: RankedProjectEntry[]
|
||||
parsedRules: ParsedRankingRule[]
|
||||
totalEligible: number
|
||||
}
|
||||
|
||||
// ─── System Prompts ────────────────────────────────────────────────────────────
|
||||
|
||||
const CRITERIA_PARSING_SYSTEM_PROMPT = `You are a ranking criteria interpreter for an ocean conservation project competition (Monaco Ocean Protection Challenge).
|
||||
|
||||
Admin will describe how they want projects ranked in natural language. Parse this into structured rules.
|
||||
|
||||
Available data fields for ranking:
|
||||
- avg_score: average jury evaluation score (1–10 scale, null if not scored)
|
||||
- pass_rate: proportion of jury members who voted to advance the project (0–1)
|
||||
- evaluator_count: number of jury members who submitted evaluations (tiebreak)
|
||||
|
||||
Return JSON only:
|
||||
{
|
||||
"rules": [
|
||||
{
|
||||
"step": 1,
|
||||
"type": "filter | sort | limit",
|
||||
"description": "Human-readable description of this rule",
|
||||
"field": "pass_rate | avg_score | evaluator_count | null",
|
||||
"operator": "gte | lte | eq | top_n | null",
|
||||
"value": <number or null>,
|
||||
"dataAvailable": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Set dataAvailable=false if the rule requires data not in the available fields list above. Do NOT invent new fields.
|
||||
Rules with dataAvailable=false will be shown as warnings to the admin — still include them.
|
||||
Order rules so filters come first, sorts next, limits last.`
|
||||
|
||||
const RANKING_SYSTEM_PROMPT = `You are a project ranking engine for an ocean conservation competition.
|
||||
|
||||
You will receive a list of anonymized projects with numeric scores and a set of parsed ranking rules.
|
||||
Apply the rules in order and return the final ranked list.
|
||||
|
||||
Return JSON only:
|
||||
{
|
||||
"ranked": [
|
||||
{
|
||||
"project_id": "P001",
|
||||
"rank": 1,
|
||||
"rationale": "Brief explanation"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Rules:
|
||||
- Apply filter rules first (remove projects that fail the filter)
|
||||
- Apply sort rules next (order remaining projects)
|
||||
- Apply limit rules last (keep only top N)
|
||||
- Projects not in the ranked output are considered excluded (not ranked last)
|
||||
- Use the project_id values exactly as given — do not change them`
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function computeCompositeScore(
|
||||
avgGlobalScore: number | null,
|
||||
passRate: number,
|
||||
evaluatorCount: number,
|
||||
maxEvaluatorCount: number,
|
||||
): number {
|
||||
const normalizedScore = avgGlobalScore != null ? (avgGlobalScore - 1) / 9 : 0.5
|
||||
const composite = normalizedScore * 0.5 + passRate * 0.5
|
||||
// Tiebreak: tiny bonus for more evaluators (won't change rank unless composite is equal)
|
||||
const tiebreakBonus = maxEvaluatorCount > 0
|
||||
? (evaluatorCount / maxEvaluatorCount) * 0.0001
|
||||
: 0
|
||||
return composite + tiebreakBonus
|
||||
}
|
||||
|
||||
function anonymizeProjectsForRanking(
|
||||
projects: ProjectForRanking[],
|
||||
): { anonymized: AnonymizedProjectForRanking[]; idMap: Map<string, string> } {
|
||||
const idMap = new Map<string, string>()
|
||||
const anonymized = projects.map((p, i) => {
|
||||
const anonId = `P${String(i + 1).padStart(3, '0')}`
|
||||
idMap.set(anonId, p.id)
|
||||
return {
|
||||
project_id: anonId,
|
||||
avg_score: p.avgGlobalScore,
|
||||
pass_rate: p.passRate,
|
||||
evaluator_count: p.evaluatorCount,
|
||||
category: p.competitionCategory,
|
||||
}
|
||||
})
|
||||
return { anonymized, idMap }
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute pass rate from Evaluation records.
|
||||
* Handles both legacy binaryDecision boolean and future dedicated field.
|
||||
* Falls back to binaryDecision if no future field exists.
|
||||
*/
|
||||
function computePassRate(evaluations: Array<{ binaryDecision: boolean | null }>): number {
|
||||
if (evaluations.length === 0) return 0
|
||||
const passCount = evaluations.filter((e) => e.binaryDecision === true).length
|
||||
return passCount / evaluations.length
|
||||
}
|
||||
|
||||
// ─── Exported Functions ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Parse natural-language ranking criteria into structured rules.
|
||||
* Returns ParsedRankingRule[] for admin review (preview mode — RANK-02, RANK-03).
|
||||
*/
|
||||
export async function parseRankingCriteria(
|
||||
criteriaText: string,
|
||||
userId?: string,
|
||||
entityId?: string,
|
||||
): Promise<ParsedRankingRule[]> {
|
||||
const { sanitized: safeCriteria } = sanitizeUserInput(criteriaText)
|
||||
|
||||
const openai = await getOpenAI()
|
||||
if (!openai) {
|
||||
throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' })
|
||||
}
|
||||
|
||||
const model = await getConfiguredModel()
|
||||
const params = buildCompletionParams(model, {
|
||||
messages: [
|
||||
{ role: 'system', content: CRITERIA_PARSING_SYSTEM_PROMPT },
|
||||
{ role: 'user', content: safeCriteria },
|
||||
],
|
||||
jsonMode: true,
|
||||
temperature: 0.1,
|
||||
maxTokens: 1000,
|
||||
})
|
||||
|
||||
let response: Awaited<ReturnType<typeof openai.chat.completions.create>>
|
||||
try {
|
||||
response = await openai.chat.completions.create(params)
|
||||
} catch (error) {
|
||||
const classified = classifyAIError(error)
|
||||
logAIError('Ranking', 'parseRankingCriteria', classified)
|
||||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message })
|
||||
}
|
||||
|
||||
const usage = extractTokenUsage(response)
|
||||
await logAIUsage({
|
||||
userId,
|
||||
action: 'RANKING',
|
||||
entityType: 'Round',
|
||||
entityId,
|
||||
model,
|
||||
promptTokens: usage.promptTokens,
|
||||
completionTokens: usage.completionTokens,
|
||||
totalTokens: usage.totalTokens,
|
||||
itemsProcessed: 1,
|
||||
status: 'SUCCESS',
|
||||
})
|
||||
|
||||
const content = response.choices[0]?.message?.content
|
||||
if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty response from AI' })
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(content) as { rules: ParsedRankingRule[] }
|
||||
return parsed.rules ?? []
|
||||
} catch {
|
||||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse AI response as JSON' })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute AI ranking for one category using pre-parsed rules.
|
||||
* Returns RankingResult with ranked project list (RANK-05, RANK-06).
|
||||
*
|
||||
* projects: raw data queried from Prisma, already filtered to one category
|
||||
* parsedRules: from parseRankingCriteria()
|
||||
*/
|
||||
export async function executeAIRanking(
|
||||
parsedRules: ParsedRankingRule[],
|
||||
projects: ProjectForRanking[],
|
||||
category: CompetitionCategory,
|
||||
userId?: string,
|
||||
entityId?: string,
|
||||
): Promise<RankingResult> {
|
||||
if (projects.length === 0) {
|
||||
return { category, rankedProjects: [], parsedRules, totalEligible: 0 }
|
||||
}
|
||||
|
||||
const maxEvaluatorCount = Math.max(...projects.map((p) => p.evaluatorCount))
|
||||
const { anonymized, idMap } = anonymizeProjectsForRanking(projects)
|
||||
|
||||
const openai = await getOpenAI()
|
||||
if (!openai) {
|
||||
throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' })
|
||||
}
|
||||
|
||||
const model = await getConfiguredModel()
|
||||
|
||||
const userPrompt = JSON.stringify({
|
||||
rules: parsedRules.filter((r) => r.dataAvailable),
|
||||
projects: anonymized,
|
||||
})
|
||||
|
||||
const params = buildCompletionParams(model, {
|
||||
messages: [
|
||||
{ role: 'system', content: RANKING_SYSTEM_PROMPT },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
jsonMode: true,
|
||||
temperature: 0,
|
||||
maxTokens: 2000,
|
||||
})
|
||||
|
||||
let response: Awaited<ReturnType<typeof openai.chat.completions.create>>
|
||||
try {
|
||||
response = await openai.chat.completions.create(params)
|
||||
} catch (error) {
|
||||
const classified = classifyAIError(error)
|
||||
logAIError('Ranking', 'executeAIRanking', classified)
|
||||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message })
|
||||
}
|
||||
|
||||
const usage = extractTokenUsage(response)
|
||||
await logAIUsage({
|
||||
userId,
|
||||
action: 'RANKING',
|
||||
entityType: 'Round',
|
||||
entityId,
|
||||
model,
|
||||
promptTokens: usage.promptTokens,
|
||||
completionTokens: usage.completionTokens,
|
||||
totalTokens: usage.totalTokens,
|
||||
itemsProcessed: projects.length,
|
||||
status: 'SUCCESS',
|
||||
})
|
||||
|
||||
const content = response.choices[0]?.message?.content
|
||||
if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty ranking response from AI' })
|
||||
|
||||
let aiRanked: Array<{ project_id: string; rank: number; rationale?: string }>
|
||||
try {
|
||||
const parsed = JSON.parse(content) as { ranked: typeof aiRanked }
|
||||
aiRanked = parsed.ranked ?? []
|
||||
} catch {
|
||||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse ranking response as JSON' })
|
||||
}
|
||||
|
||||
// Build a lookup by anonymousId for project data
|
||||
const projectByAnonId = new Map(
|
||||
anonymized.map((a) => [a.project_id, projects.find((p) => p.id === idMap.get(a.project_id))!])
|
||||
)
|
||||
|
||||
const rankedProjects: RankedProjectEntry[] = aiRanked
|
||||
.filter((entry) => idMap.has(entry.project_id))
|
||||
.map((entry) => {
|
||||
const realId = idMap.get(entry.project_id)!
|
||||
const proj = projectByAnonId.get(entry.project_id)!
|
||||
return {
|
||||
projectId: realId,
|
||||
rank: entry.rank,
|
||||
compositeScore: computeCompositeScore(
|
||||
proj.avgGlobalScore,
|
||||
proj.passRate,
|
||||
proj.evaluatorCount,
|
||||
maxEvaluatorCount,
|
||||
),
|
||||
avgGlobalScore: proj.avgGlobalScore,
|
||||
passRate: proj.passRate,
|
||||
evaluatorCount: proj.evaluatorCount,
|
||||
aiRationale: entry.rationale,
|
||||
}
|
||||
})
|
||||
.sort((a, b) => a.rank - b.rank)
|
||||
|
||||
return {
|
||||
category,
|
||||
rankedProjects,
|
||||
parsedRules,
|
||||
totalEligible: projects.length,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Quick-rank: parse criteria and execute ranking in one step.
|
||||
* Returns results for all categories (RANK-04).
|
||||
* The prisma parameter is used to fetch project evaluation data.
|
||||
*/
|
||||
export async function quickRank(
|
||||
criteriaText: string,
|
||||
roundId: string,
|
||||
prisma: PrismaClient,
|
||||
userId?: string,
|
||||
): Promise<{ startup: RankingResult; concept: RankingResult; parsedRules: ParsedRankingRule[] }> {
|
||||
const parsedRules = await parseRankingCriteria(criteriaText, userId, roundId)
|
||||
const [startup, concept] = await Promise.all([
|
||||
fetchAndRankCategory('STARTUP', parsedRules, roundId, prisma, userId),
|
||||
fetchAndRankCategory('BUSINESS_CONCEPT', parsedRules, roundId, prisma, userId),
|
||||
])
|
||||
return { startup, concept, parsedRules }
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper: fetch eligible projects for one category and execute ranking.
|
||||
* Excluded: withdrawn projects and projects with zero submitted evaluations (locked decision).
|
||||
*
|
||||
* Exported so the tRPC router can call it separately when executing pre-parsed rules.
|
||||
*/
|
||||
export async function fetchAndRankCategory(
|
||||
category: CompetitionCategory,
|
||||
parsedRules: ParsedRankingRule[],
|
||||
roundId: string,
|
||||
prisma: PrismaClient,
|
||||
userId?: string,
|
||||
): Promise<RankingResult> {
|
||||
// Query submitted evaluations grouped by projectId for this category
|
||||
const assignments = await prisma.assignment.findMany({
|
||||
where: {
|
||||
roundId,
|
||||
isRequired: true,
|
||||
project: {
|
||||
competitionCategory: category,
|
||||
// Exclude withdrawn projects
|
||||
projectRoundStates: {
|
||||
none: { roundId, state: 'WITHDRAWN' },
|
||||
},
|
||||
},
|
||||
evaluation: {
|
||||
status: 'SUBMITTED', // Only count completed evaluations
|
||||
},
|
||||
},
|
||||
include: {
|
||||
evaluation: {
|
||||
select: { globalScore: true, binaryDecision: true },
|
||||
},
|
||||
project: {
|
||||
select: { id: true, competitionCategory: true },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// Group by projectId
|
||||
const byProject = new Map<string, Array<{ globalScore: number | null; binaryDecision: boolean | null }>>()
|
||||
for (const a of assignments) {
|
||||
if (!a.evaluation) continue
|
||||
const list = byProject.get(a.project.id) ?? []
|
||||
list.push({ globalScore: a.evaluation.globalScore, binaryDecision: a.evaluation.binaryDecision })
|
||||
byProject.set(a.project.id, list)
|
||||
}
|
||||
|
||||
// Build ProjectForRanking, excluding projects with zero submitted evaluations
|
||||
const projects: ProjectForRanking[] = []
|
||||
for (const [projectId, evals] of byProject.entries()) {
|
||||
if (evals.length === 0) continue // Exclude: no submitted evaluations
|
||||
const avgGlobalScore = evals.some((e) => e.globalScore != null)
|
||||
? evals.filter((e) => e.globalScore != null).reduce((sum, e) => sum + e.globalScore!, 0) /
|
||||
evals.filter((e) => e.globalScore != null).length
|
||||
: null
|
||||
const passRate = computePassRate(evals)
|
||||
projects.push({ id: projectId, competitionCategory: category, avgGlobalScore, passRate, evaluatorCount: evals.length })
|
||||
}
|
||||
|
||||
return executeAIRanking(parsedRules, projects, category, userId, roundId)
|
||||
}
|
||||
Reference in New Issue
Block a user