be4449e4ef
All checks were successful
Build and Push Docker Image / build (push) Successful in 8m7s
The ranking dashboard showed 0/X Yes for every project in rounds using the 'advance' criterion type because the matcher only looked for type === 'boolean' with a "move to the next stage" label. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
869 lines
32 KiB
TypeScript
869 lines
32 KiB
TypeScript
/**
|
||
* AI Ranking Service
|
||
*
|
||
* Parses natural-language ranking criteria into structured rules and
|
||
* executes per-category project ranking using OpenAI.
|
||
*
|
||
* GDPR Compliance:
|
||
* - All project data is anonymized before AI processing (P001, P002, …)
|
||
* - No personal identifiers or real project IDs in prompts or responses
|
||
*
|
||
* Design decisions:
|
||
* - Per-category processing (STARTUP / BUSINESS_CONCEPT) — two parallel AI calls
|
||
* - Projects with zero submitted evaluations are excluded (not ranked last)
|
||
* - compositeScore uses weighted criteria when available, falls back to avgGlobalScore
|
||
* - Z-score normalization corrects for juror grading bias
|
||
*/
|
||
|
||
import { getOpenAI, getConfiguredModel, buildCompletionParams } from '@/lib/openai'
|
||
import { logAIUsage, extractTokenUsage } from '@/server/utils/ai-usage'
|
||
import { classifyAIError, logAIError } from './ai-errors'
|
||
import { findActiveForm } from '@/server/utils/evaluation-form-lookup'
|
||
import { sanitizeUserInput } from '@/server/services/ai-prompt-guard'
|
||
import { TRPCError } from '@trpc/server'
|
||
import type { CompetitionCategory, PrismaClient } from '@prisma/client'
|
||
import type { EvaluationConfig } from '@/types/competition-configs'
|
||
|
||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||
|
||
// Criterion definition from EvaluationForm.criteriaJson
|
||
interface CriterionDef {
|
||
id: string
|
||
label: string
|
||
type?: string
|
||
scale?: number | string
|
||
weight?: number
|
||
}
|
||
|
||
// Internal shape of a project before anonymization
|
||
interface ProjectForRanking {
|
||
id: string
|
||
competitionCategory: CompetitionCategory
|
||
avgGlobalScore: number | null // average of submitted Evaluation.globalScore
|
||
normalizedAvgScore: number | null // z-score normalized average
|
||
passRate: number // proportion of binaryDecision=true among SUBMITTED evaluations
|
||
evaluatorCount: number // count of SUBMITTED evaluations
|
||
criterionAverages: Record<string, number> // criterionId → raw average score
|
||
normalizedCriterionAverages: Record<string, number> // criterionId → z-score normalized average
|
||
}
|
||
|
||
// Anonymized shape sent to OpenAI
|
||
interface AnonymizedProjectForRanking {
|
||
project_id: string // "P001", "P002", etc. — never real IDs
|
||
avg_score: number | null
|
||
normalized_avg_score: number | null
|
||
pass_rate: number // 0–1
|
||
evaluator_count: number
|
||
category: string
|
||
criteria_scores: Record<string, number>
|
||
normalized_criteria_scores: Record<string, number>
|
||
}
|
||
|
||
// Criterion definition sent to OpenAI
|
||
interface CriterionDefForAI {
|
||
name: string
|
||
weight: number
|
||
scale: string
|
||
}
|
||
|
||
// A single parsed rule returned by the criteria parser
|
||
export interface ParsedRankingRule {
|
||
step: number
|
||
type: 'filter' | 'sort' | 'limit'
|
||
description: string // Human-readable rule text
|
||
field: 'pass_rate' | 'avg_score' | 'evaluator_count' | null
|
||
operator: 'gte' | 'lte' | 'eq' | 'top_n' | null
|
||
value: number | null
|
||
dataAvailable: boolean // false = rule references unavailable data; UI should warn
|
||
}
|
||
|
||
// A single project entry in the ranked output
|
||
export interface RankedProjectEntry {
|
||
projectId: string // Real project ID (de-anonymized)
|
||
rank: number // 1-indexed
|
||
compositeScore: number // 0–1 floating point
|
||
avgGlobalScore: number | null
|
||
normalizedAvgScore: number | null
|
||
passRate: number
|
||
evaluatorCount: number
|
||
aiRationale?: string // Optional: AI explanation for this project's rank
|
||
}
|
||
|
||
// Full result for one category
|
||
export interface RankingResult {
|
||
category: CompetitionCategory
|
||
rankedProjects: RankedProjectEntry[]
|
||
parsedRules: ParsedRankingRule[]
|
||
totalEligible: number
|
||
}
|
||
|
||
// ─── System Prompts ────────────────────────────────────────────────────────────
|
||
|
||
const CRITERIA_PARSING_SYSTEM_PROMPT = `You are a ranking criteria interpreter for an ocean conservation project competition (Monaco Ocean Protection Challenge).
|
||
|
||
Admin will describe how they want projects ranked in natural language. Parse this into structured rules.
|
||
|
||
Available data fields for ranking:
|
||
- avg_score: average jury evaluation score (1–10 scale, null if not scored)
|
||
- normalized_avg_score: bias-corrected average (z-score normalized across jurors)
|
||
- pass_rate: proportion of jury members who voted to advance the project (0–1)
|
||
- evaluator_count: number of jury members who submitted evaluations (tiebreak)
|
||
- criteria_scores: per-criterion averages (keyed by criterion name)
|
||
- normalized_criteria_scores: bias-corrected per-criterion averages
|
||
|
||
Return JSON only:
|
||
{
|
||
"rules": [
|
||
{
|
||
"step": 1,
|
||
"type": "filter | sort | limit",
|
||
"description": "Human-readable description of this rule",
|
||
"field": "pass_rate | avg_score | evaluator_count | null",
|
||
"operator": "gte | lte | eq | top_n | null",
|
||
"value": <number or null>,
|
||
"dataAvailable": true
|
||
}
|
||
]
|
||
}
|
||
|
||
Set dataAvailable=false if the rule requires data not in the available fields list above. Do NOT invent new fields.
|
||
Rules with dataAvailable=false will be shown as warnings to the admin — still include them.
|
||
Order rules so filters come first, sorts next, limits last.`
|
||
|
||
const RANKING_SYSTEM_PROMPT = `You are a project ranking engine for an ocean conservation competition.
|
||
|
||
You will receive:
|
||
1. A list of anonymized projects with numeric scores (including per-criterion averages and bias-corrected scores)
|
||
2. A set of parsed ranking rules
|
||
3. Optional: criteria_definitions with weights indicating the relative importance of each evaluation criterion
|
||
|
||
When criteria_definitions with weights are provided, use the weighted criteria scores as a PRIMARY ranking factor.
|
||
The weighted score is: sum(criterion_avg * weight) / sum(weights).
|
||
Use normalized (bias-corrected) scores when available — they account for differences in juror grading harshness.
|
||
|
||
Apply the rules in order and return the final ranked list.
|
||
|
||
Return JSON only:
|
||
{
|
||
"ranked": [
|
||
{
|
||
"project_id": "P001",
|
||
"rank": 1,
|
||
"rationale": "Brief explanation"
|
||
}
|
||
]
|
||
}
|
||
|
||
CRITICAL Rules:
|
||
- You MUST include EVERY project in the ranked output — never exclude or filter out any project
|
||
- Apply sort rules to determine the ranking order
|
||
- If filter criteria exist, use them to inform ranking priority (projects meeting all criteria rank higher, those failing criteria rank lower) but still include ALL projects
|
||
- Ignore any limit rules — always return all projects
|
||
- Use the project_id values exactly as given — do not change them
|
||
- Ranks must be contiguous (1, 2, 3, …) with no gaps`
|
||
|
||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Compute composite score using weighted criteria if available,
|
||
* falling back to avgGlobalScore otherwise.
|
||
*/
|
||
function computeCompositeScore(
|
||
project: ProjectForRanking,
|
||
maxEvaluatorCount: number,
|
||
criteriaWeights: Record<string, number> | undefined,
|
||
criterionDefs: CriterionDef[],
|
||
scoreWeight = 5,
|
||
passRateWeight = 5,
|
||
): number {
|
||
let scoreComponent: number
|
||
|
||
// Try weighted criteria first
|
||
if (criteriaWeights && Object.keys(criteriaWeights).length > 0) {
|
||
let weightedSum = 0
|
||
let totalWeight = 0
|
||
for (const [criterionId, weight] of Object.entries(criteriaWeights)) {
|
||
if (weight <= 0) continue
|
||
// Use normalized scores if available, otherwise raw
|
||
const score = project.normalizedCriterionAverages[criterionId]
|
||
?? project.criterionAverages[criterionId]
|
||
if (score == null) continue
|
||
// Normalize to 0–1 based on criterion scale
|
||
const def = criterionDefs.find((d) => d.id === criterionId)
|
||
const maxScale = typeof def?.scale === 'number' ? def.scale
|
||
: typeof def?.scale === 'string' ? parseInt(def.scale.split('-').pop() ?? '10', 10)
|
||
: 10
|
||
const normalizedScore = maxScale > 1 ? (score - 1) / (maxScale - 1) : score
|
||
weightedSum += normalizedScore * weight
|
||
totalWeight += weight
|
||
}
|
||
scoreComponent = totalWeight > 0 ? weightedSum / totalWeight : 0.5
|
||
} else {
|
||
// Fallback: use avgGlobalScore normalized to 0–1
|
||
const avg = project.normalizedAvgScore ?? project.avgGlobalScore
|
||
scoreComponent = avg != null ? (avg - 1) / 9 : 0.5
|
||
}
|
||
|
||
// Configurable score vs pass-rate weighting
|
||
const totalW = scoreWeight + passRateWeight
|
||
const sW = totalW > 0 ? scoreWeight / totalW : 0.5
|
||
const pW = totalW > 0 ? passRateWeight / totalW : 0.5
|
||
const composite = scoreComponent * sW + project.passRate * pW
|
||
|
||
// Tiebreak: tiny bonus for more evaluators (won't change rank unless composite is equal)
|
||
const tiebreakBonus = maxEvaluatorCount > 0
|
||
? (project.evaluatorCount / maxEvaluatorCount) * 0.0001
|
||
: 0
|
||
return composite + tiebreakBonus
|
||
}
|
||
|
||
function anonymizeProjectsForRanking(
|
||
projects: ProjectForRanking[],
|
||
criterionDefs: CriterionDef[],
|
||
): { anonymized: AnonymizedProjectForRanking[]; idMap: Map<string, string> } {
|
||
// Build id → label map for criterion names (anonymize IDs)
|
||
const idToLabel = new Map(criterionDefs.map((d) => [d.id, d.label]))
|
||
|
||
const idMap = new Map<string, string>()
|
||
const anonymized = projects.map((p, i) => {
|
||
const anonId = `P${String(i + 1).padStart(3, '0')}`
|
||
idMap.set(anonId, p.id)
|
||
|
||
// Convert criterion ID keys to human-readable labels
|
||
const criteriaScores: Record<string, number> = {}
|
||
for (const [id, score] of Object.entries(p.criterionAverages)) {
|
||
const label = idToLabel.get(id) ?? id
|
||
criteriaScores[label] = Math.round(score * 100) / 100
|
||
}
|
||
const normalizedCriteriaScores: Record<string, number> = {}
|
||
for (const [id, score] of Object.entries(p.normalizedCriterionAverages)) {
|
||
const label = idToLabel.get(id) ?? id
|
||
normalizedCriteriaScores[label] = Math.round(score * 100) / 100
|
||
}
|
||
|
||
return {
|
||
project_id: anonId,
|
||
avg_score: p.avgGlobalScore != null ? Math.round(p.avgGlobalScore * 100) / 100 : null,
|
||
normalized_avg_score: p.normalizedAvgScore != null ? Math.round(p.normalizedAvgScore * 100) / 100 : null,
|
||
pass_rate: p.passRate,
|
||
evaluator_count: p.evaluatorCount,
|
||
category: p.competitionCategory,
|
||
criteria_scores: criteriaScores,
|
||
normalized_criteria_scores: normalizedCriteriaScores,
|
||
}
|
||
})
|
||
return { anonymized, idMap }
|
||
}
|
||
|
||
/**
|
||
* Find the advance-decision criterion ID from criteria definitions.
|
||
* Current forms use type === 'advance' (one allowed per form). Legacy forms used
|
||
* type === 'boolean' with a "move to the next stage" label.
|
||
* Accepts criteria from EvaluationForm.criteriaJson (preferred) or round configJson (legacy).
|
||
*/
|
||
function findBooleanCriterionId(criterionDefs: Array<{ id: string; label: string; type?: string }>): string | null {
|
||
const advanceCriterion = criterionDefs.find((c) => c.type === 'advance')
|
||
if (advanceCriterion) return advanceCriterion.id
|
||
const boolCriterion = criterionDefs.find(
|
||
(c) => c.type === 'boolean' && c.label?.toLowerCase().includes('move to the next stage'),
|
||
)
|
||
return boolCriterion?.id ?? null
|
||
}
|
||
|
||
/**
|
||
* Resolve the binary advance decision for an evaluation.
|
||
* 1. Use binaryDecision column if set
|
||
* 2. Fall back to the boolean criterion in criterionScoresJson
|
||
*/
|
||
function resolveBinaryDecision(
|
||
binaryDecision: boolean | null,
|
||
criterionScoresJson: Record<string, unknown> | null,
|
||
boolCriterionId: string | null,
|
||
): boolean | null {
|
||
if (binaryDecision != null) return binaryDecision
|
||
if (!boolCriterionId || !criterionScoresJson) return null
|
||
const value = criterionScoresJson[boolCriterionId]
|
||
if (typeof value === 'boolean') return value
|
||
if (value === 'true') return true
|
||
if (value === 'false') return false
|
||
return null
|
||
}
|
||
|
||
/**
|
||
* Compute pass rate from Evaluation records.
|
||
* Counts evaluations where the advance decision resolved to true.
|
||
* Evaluations with null decision are treated as "no" (not counted as pass).
|
||
*/
|
||
function computePassRate(evaluations: Array<{ resolvedDecision: boolean | null }>): number {
|
||
if (evaluations.length === 0) return 0
|
||
const passCount = evaluations.filter((e) => e.resolvedDecision === true).length
|
||
return passCount / evaluations.length
|
||
}
|
||
|
||
// ─── Z-Score Normalization ──────────────────────────────────────────────────
|
||
|
||
interface JurorStats {
|
||
mean: number
|
||
stddev: number
|
||
count: number
|
||
}
|
||
|
||
/**
|
||
* Compute per-juror grading statistics (mean and stddev) for z-score normalization.
|
||
* Only considers numeric criterion scores and globalScore from SUBMITTED evaluations.
|
||
*/
|
||
function computeJurorStats(
|
||
assignments: Array<{
|
||
userId: string
|
||
evaluation: {
|
||
globalScore: number | null
|
||
criterionScoresJson: Record<string, unknown> | null
|
||
} | null
|
||
}>,
|
||
numericCriterionIds: Set<string>,
|
||
): Map<string, JurorStats> {
|
||
// Collect all numeric scores per juror
|
||
const jurorScores = new Map<string, number[]>()
|
||
for (const a of assignments) {
|
||
if (!a.evaluation) continue
|
||
const scores: number[] = []
|
||
if (a.evaluation.globalScore != null) scores.push(a.evaluation.globalScore)
|
||
if (a.evaluation.criterionScoresJson) {
|
||
for (const [id, val] of Object.entries(a.evaluation.criterionScoresJson)) {
|
||
if (numericCriterionIds.has(id) && typeof val === 'number') {
|
||
scores.push(val)
|
||
}
|
||
}
|
||
}
|
||
const existing = jurorScores.get(a.userId) ?? []
|
||
existing.push(...scores)
|
||
jurorScores.set(a.userId, existing)
|
||
}
|
||
|
||
const stats = new Map<string, JurorStats>()
|
||
for (const [userId, scores] of jurorScores.entries()) {
|
||
if (scores.length < 2) {
|
||
// Not enough data for meaningful normalization — skip
|
||
stats.set(userId, { mean: 0, stddev: 0, count: scores.length })
|
||
continue
|
||
}
|
||
const mean = scores.reduce((a, b) => a + b, 0) / scores.length
|
||
const variance = scores.reduce((sum, s) => sum + (s - mean) ** 2, 0) / scores.length
|
||
const stddev = Math.sqrt(variance)
|
||
stats.set(userId, { mean, stddev, count: scores.length })
|
||
}
|
||
return stats
|
||
}
|
||
|
||
/**
|
||
* Normalize a raw score using z-score normalization.
|
||
* Returns the z-score, or null if normalization isn't possible (too few evals or stddev=0).
|
||
*/
|
||
function zScoreNormalize(raw: number, stats: JurorStats): number | null {
|
||
if (stats.count < 2 || stats.stddev === 0) return null
|
||
return (raw - stats.mean) / stats.stddev
|
||
}
|
||
|
||
// ─── Exported Functions ───────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* Parse natural-language ranking criteria into structured rules.
|
||
* Returns ParsedRankingRule[] for admin review (preview mode — RANK-02, RANK-03).
|
||
*/
|
||
export async function parseRankingCriteria(
|
||
criteriaText: string,
|
||
userId?: string,
|
||
entityId?: string,
|
||
): Promise<ParsedRankingRule[]> {
|
||
const { sanitized: safeCriteria } = sanitizeUserInput(criteriaText)
|
||
|
||
const openai = await getOpenAI()
|
||
if (!openai) {
|
||
throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' })
|
||
}
|
||
|
||
const model = await getConfiguredModel()
|
||
const params = buildCompletionParams(model, {
|
||
messages: [
|
||
{ role: 'system', content: CRITERIA_PARSING_SYSTEM_PROMPT },
|
||
{ role: 'user', content: safeCriteria },
|
||
],
|
||
jsonMode: true,
|
||
temperature: 0.1,
|
||
maxTokens: 1000,
|
||
})
|
||
|
||
let response: Awaited<ReturnType<typeof openai.chat.completions.create>>
|
||
try {
|
||
response = await openai.chat.completions.create(params)
|
||
} catch (error) {
|
||
const classified = classifyAIError(error)
|
||
logAIError('Ranking', 'parseRankingCriteria', classified)
|
||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message })
|
||
}
|
||
|
||
const usage = extractTokenUsage(response)
|
||
await logAIUsage({
|
||
userId,
|
||
action: 'RANKING',
|
||
entityType: 'Round',
|
||
entityId,
|
||
model,
|
||
promptTokens: usage.promptTokens,
|
||
completionTokens: usage.completionTokens,
|
||
totalTokens: usage.totalTokens,
|
||
itemsProcessed: 1,
|
||
status: 'SUCCESS',
|
||
})
|
||
|
||
const content = response.choices[0]?.message?.content
|
||
if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty response from AI' })
|
||
|
||
try {
|
||
const parsed = JSON.parse(content) as { rules: ParsedRankingRule[] }
|
||
return parsed.rules ?? []
|
||
} catch {
|
||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse AI response as JSON' })
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Execute AI ranking for one category using pre-parsed rules.
|
||
* Returns RankingResult with ranked project list (RANK-05, RANK-06).
|
||
*
|
||
* projects: raw data queried from Prisma, already filtered to one category
|
||
* parsedRules: from parseRankingCriteria()
|
||
* criteriaWeights: optional admin-configured weights from round config
|
||
* criterionDefs: criterion definitions from the evaluation form
|
||
*/
|
||
export async function executeAIRanking(
|
||
parsedRules: ParsedRankingRule[],
|
||
projects: ProjectForRanking[],
|
||
category: CompetitionCategory,
|
||
criteriaWeights: Record<string, number> | undefined,
|
||
criterionDefs: CriterionDef[],
|
||
userId?: string,
|
||
entityId?: string,
|
||
scoreWeight = 5,
|
||
passRateWeight = 5,
|
||
): Promise<RankingResult> {
|
||
if (projects.length === 0) {
|
||
return { category, rankedProjects: [], parsedRules, totalEligible: 0 }
|
||
}
|
||
|
||
const maxEvaluatorCount = Math.max(...projects.map((p) => p.evaluatorCount))
|
||
const { anonymized, idMap } = anonymizeProjectsForRanking(projects, criterionDefs)
|
||
|
||
const openai = await getOpenAI()
|
||
if (!openai) {
|
||
throw new TRPCError({ code: 'PRECONDITION_FAILED', message: 'OpenAI not configured' })
|
||
}
|
||
|
||
const model = await getConfiguredModel()
|
||
|
||
// Build criteria_definitions for the AI prompt (only numeric criteria)
|
||
const numericDefs = criterionDefs.filter((d) => !d.type || d.type === 'numeric')
|
||
const criteriaDefsForAI: CriterionDefForAI[] = numericDefs.map((d) => {
|
||
const adminWeight = criteriaWeights?.[d.id] ?? d.weight ?? 1
|
||
const scale = typeof d.scale === 'number' ? `1-${d.scale}` : typeof d.scale === 'string' ? d.scale : '1-10'
|
||
return { name: d.label, weight: adminWeight, scale }
|
||
})
|
||
|
||
const promptData: Record<string, unknown> = {
|
||
rules: parsedRules.filter((r) => r.dataAvailable),
|
||
projects: anonymized,
|
||
}
|
||
if (criteriaDefsForAI.length > 0) {
|
||
promptData.criteria_definitions = criteriaDefsForAI
|
||
}
|
||
|
||
const userPrompt = JSON.stringify(promptData)
|
||
|
||
const params = buildCompletionParams(model, {
|
||
messages: [
|
||
{ role: 'system', content: RANKING_SYSTEM_PROMPT },
|
||
{ role: 'user', content: userPrompt },
|
||
],
|
||
jsonMode: true,
|
||
temperature: 0,
|
||
// ~50 tokens per project entry; scale for large pools with generous buffer
|
||
maxTokens: Math.max(2000, projects.length * 80),
|
||
})
|
||
|
||
let response: Awaited<ReturnType<typeof openai.chat.completions.create>>
|
||
try {
|
||
response = await openai.chat.completions.create(params)
|
||
} catch (error) {
|
||
const classified = classifyAIError(error)
|
||
logAIError('Ranking', 'executeAIRanking', classified)
|
||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: classified.message })
|
||
}
|
||
|
||
const usage = extractTokenUsage(response)
|
||
await logAIUsage({
|
||
userId,
|
||
action: 'RANKING',
|
||
entityType: 'Round',
|
||
entityId,
|
||
model,
|
||
promptTokens: usage.promptTokens,
|
||
completionTokens: usage.completionTokens,
|
||
totalTokens: usage.totalTokens,
|
||
itemsProcessed: projects.length,
|
||
status: 'SUCCESS',
|
||
})
|
||
|
||
const content = response.choices[0]?.message?.content
|
||
if (!content) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Empty ranking response from AI' })
|
||
|
||
let aiRanked: Array<{ project_id: string; rank: number; rationale?: string }>
|
||
try {
|
||
const parsed = JSON.parse(content) as { ranked: typeof aiRanked }
|
||
aiRanked = parsed.ranked ?? []
|
||
} catch {
|
||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to parse ranking response as JSON' })
|
||
}
|
||
|
||
// Deduplicate AI response — keep only the first occurrence of each project_id
|
||
const seenAnonIds = new Set<string>()
|
||
aiRanked = aiRanked.filter((entry) => {
|
||
if (seenAnonIds.has(entry.project_id)) return false
|
||
seenAnonIds.add(entry.project_id)
|
||
return true
|
||
})
|
||
|
||
// Build a lookup by anonymousId for project data
|
||
const projectByAnonId = new Map(
|
||
anonymized.map((a) => [a.project_id, projects.find((p) => p.id === idMap.get(a.project_id))!])
|
||
)
|
||
|
||
const rankedProjects: RankedProjectEntry[] = aiRanked
|
||
.filter((entry) => idMap.has(entry.project_id))
|
||
.map((entry) => {
|
||
const realId = idMap.get(entry.project_id)!
|
||
const proj = projectByAnonId.get(entry.project_id)!
|
||
return {
|
||
projectId: realId,
|
||
rank: entry.rank,
|
||
compositeScore: computeCompositeScore(proj, maxEvaluatorCount, criteriaWeights, criterionDefs, scoreWeight, passRateWeight),
|
||
avgGlobalScore: proj.avgGlobalScore,
|
||
normalizedAvgScore: proj.normalizedAvgScore,
|
||
passRate: proj.passRate,
|
||
evaluatorCount: proj.evaluatorCount,
|
||
aiRationale: entry.rationale,
|
||
}
|
||
})
|
||
.sort((a, b) => a.rank - b.rank)
|
||
|
||
// ─── Ensure ALL projects are included (AI may omit some due to token limits) ──
|
||
const rankedIds = new Set(rankedProjects.map((r) => r.projectId))
|
||
const unrankedProjects = projects
|
||
.filter((p) => !rankedIds.has(p.id))
|
||
.map((p) => ({
|
||
projectId: p.id,
|
||
rank: 0,
|
||
compositeScore: computeCompositeScore(p, maxEvaluatorCount, criteriaWeights, criterionDefs),
|
||
avgGlobalScore: p.avgGlobalScore,
|
||
normalizedAvgScore: p.normalizedAvgScore,
|
||
passRate: p.passRate,
|
||
evaluatorCount: p.evaluatorCount,
|
||
}))
|
||
.sort((a, b) => b.compositeScore - a.compositeScore)
|
||
|
||
for (const proj of unrankedProjects) {
|
||
rankedProjects.push(proj)
|
||
}
|
||
|
||
// Sort ALL projects by compositeScore descending (deterministic, score-based order).
|
||
// The compositeScore incorporates weighted criteria, z-score normalization, pass rate,
|
||
// and evaluator count — so highest-rated projects always appear first.
|
||
rankedProjects.sort((a, b) => b.compositeScore - a.compositeScore)
|
||
|
||
// Re-number ranks to be contiguous (1, 2, 3, …)
|
||
rankedProjects.forEach((p, i) => { p.rank = i + 1 })
|
||
|
||
return {
|
||
category,
|
||
rankedProjects,
|
||
parsedRules,
|
||
totalEligible: projects.length,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Quick-rank: parse criteria and execute ranking in one step.
|
||
* Returns results for all categories (RANK-04).
|
||
* The prisma parameter is used to fetch project evaluation data.
|
||
*/
|
||
export async function quickRank(
|
||
criteriaText: string,
|
||
roundId: string,
|
||
prisma: PrismaClient,
|
||
userId?: string,
|
||
): Promise<{ startup: RankingResult; concept: RankingResult; parsedRules: ParsedRankingRule[] }> {
|
||
const parsedRules = await parseRankingCriteria(criteriaText, userId, roundId)
|
||
const [startup, concept] = await Promise.all([
|
||
fetchAndRankCategory('STARTUP', parsedRules, roundId, prisma, userId),
|
||
fetchAndRankCategory('BUSINESS_CONCEPT', parsedRules, roundId, prisma, userId),
|
||
])
|
||
return { startup, concept, parsedRules }
|
||
}
|
||
|
||
// Result of fetchCategoryProjects — shared data for both AI and formula ranking
|
||
interface CategoryProjectData {
|
||
projects: ProjectForRanking[]
|
||
criteriaWeights: Record<string, number> | undefined
|
||
criterionDefs: CriterionDef[]
|
||
scoreWeight: number
|
||
passRateWeight: number
|
||
}
|
||
|
||
/**
|
||
* Shared data-gathering helper: fetch eligible projects for one category.
|
||
* Handles: round config + eval form loading, z-score normalization,
|
||
* per-criterion averages, pass rates, etc.
|
||
*
|
||
* Used by both `fetchAndRankCategory` (AI path) and `formulaRankCategory` (formula path).
|
||
*/
|
||
async function fetchCategoryProjects(
|
||
category: CompetitionCategory,
|
||
roundId: string,
|
||
prisma: PrismaClient,
|
||
): Promise<CategoryProjectData> {
|
||
// Fetch the round config and evaluation form in parallel (category-aware)
|
||
const [round, evalForm] = await Promise.all([
|
||
prisma.round.findUniqueOrThrow({
|
||
where: { id: roundId },
|
||
select: { configJson: true },
|
||
}),
|
||
findActiveForm(prisma, roundId, category),
|
||
])
|
||
|
||
const roundConfig = round.configJson as Record<string, unknown> | null
|
||
|
||
// Parse evaluation config for criteria weights and formula weights
|
||
const evalConfig = roundConfig as EvaluationConfig | null
|
||
const criteriaWeights = evalConfig?.criteriaWeights ?? undefined
|
||
const scoreWeight = evalConfig?.scoreWeight ?? 5
|
||
const passRateWeight = evalConfig?.passRateWeight ?? 5
|
||
|
||
// Parse criterion definitions from the evaluation form
|
||
const criterionDefs: CriterionDef[] = evalForm?.criteriaJson
|
||
? (evalForm.criteriaJson as unknown as CriterionDef[])
|
||
: []
|
||
const boolCriterionId = findBooleanCriterionId(criterionDefs)
|
||
const numericCriterionIds = new Set(
|
||
criterionDefs.filter((d) => !d.type || d.type === 'numeric').map((d) => d.id),
|
||
)
|
||
|
||
// Query submitted evaluations grouped by projectId for this category
|
||
const assignments = await prisma.assignment.findMany({
|
||
where: {
|
||
roundId,
|
||
isRequired: true,
|
||
project: {
|
||
competitionCategory: category,
|
||
// Exclude withdrawn projects
|
||
projectRoundStates: {
|
||
none: { roundId, state: 'WITHDRAWN' },
|
||
},
|
||
},
|
||
evaluation: {
|
||
status: 'SUBMITTED', // Only count completed evaluations
|
||
},
|
||
},
|
||
include: {
|
||
evaluation: {
|
||
select: { globalScore: true, binaryDecision: true, criterionScoresJson: true },
|
||
},
|
||
project: {
|
||
select: { id: true, competitionCategory: true },
|
||
},
|
||
},
|
||
})
|
||
|
||
// Compute per-juror stats for z-score normalization
|
||
const jurorStats = computeJurorStats(
|
||
assignments.map((a) => ({
|
||
userId: a.userId,
|
||
evaluation: a.evaluation ? {
|
||
globalScore: a.evaluation.globalScore,
|
||
criterionScoresJson: a.evaluation.criterionScoresJson as Record<string, unknown> | null,
|
||
} : null,
|
||
})),
|
||
numericCriterionIds,
|
||
)
|
||
|
||
// Group by projectId, collect per-juror scores for aggregation
|
||
type EvalData = {
|
||
globalScore: number | null
|
||
resolvedDecision: boolean | null
|
||
criterionScores: Record<string, unknown> | null
|
||
userId: string
|
||
}
|
||
const byProject = new Map<string, EvalData[]>()
|
||
for (const a of assignments) {
|
||
if (!a.evaluation) continue
|
||
const resolved = resolveBinaryDecision(
|
||
a.evaluation.binaryDecision,
|
||
a.evaluation.criterionScoresJson as Record<string, unknown> | null,
|
||
boolCriterionId,
|
||
)
|
||
const list = byProject.get(a.project.id) ?? []
|
||
list.push({
|
||
globalScore: a.evaluation.globalScore,
|
||
resolvedDecision: resolved,
|
||
criterionScores: a.evaluation.criterionScoresJson as Record<string, unknown> | null,
|
||
userId: a.userId,
|
||
})
|
||
byProject.set(a.project.id, list)
|
||
}
|
||
|
||
// Build ProjectForRanking, excluding projects with zero submitted evaluations
|
||
const projects: ProjectForRanking[] = []
|
||
for (const [projectId, evals] of byProject.entries()) {
|
||
if (evals.length === 0) continue
|
||
|
||
// Raw avg global score
|
||
const avgGlobalScore = evals.some((e) => e.globalScore != null)
|
||
? evals.filter((e) => e.globalScore != null).reduce((sum, e) => sum + e.globalScore!, 0) /
|
||
evals.filter((e) => e.globalScore != null).length
|
||
: null
|
||
|
||
// Z-score normalized avg global score
|
||
const normalizedGlobalScores: number[] = []
|
||
for (const e of evals) {
|
||
if (e.globalScore == null) continue
|
||
const stats = jurorStats.get(e.userId)
|
||
if (!stats) continue
|
||
const z = zScoreNormalize(e.globalScore, stats)
|
||
if (z != null) normalizedGlobalScores.push(z)
|
||
}
|
||
const normalizedAvgScore = normalizedGlobalScores.length > 0
|
||
? normalizedGlobalScores.reduce((a, b) => a + b, 0) / normalizedGlobalScores.length
|
||
: null
|
||
|
||
// Per-criterion raw averages (numeric criteria only)
|
||
const criterionAverages: Record<string, number> = {}
|
||
for (const criterionId of numericCriterionIds) {
|
||
const values: number[] = []
|
||
for (const e of evals) {
|
||
if (!e.criterionScores) continue
|
||
const val = e.criterionScores[criterionId]
|
||
if (typeof val === 'number') values.push(val)
|
||
}
|
||
if (values.length > 0) {
|
||
criterionAverages[criterionId] = values.reduce((a, b) => a + b, 0) / values.length
|
||
}
|
||
}
|
||
|
||
// Per-criterion z-score normalized averages
|
||
const normalizedCriterionAverages: Record<string, number> = {}
|
||
for (const criterionId of numericCriterionIds) {
|
||
const zScores: number[] = []
|
||
for (const e of evals) {
|
||
if (!e.criterionScores) continue
|
||
const val = e.criterionScores[criterionId]
|
||
if (typeof val !== 'number') continue
|
||
const stats = jurorStats.get(e.userId)
|
||
if (!stats) continue
|
||
const z = zScoreNormalize(val, stats)
|
||
if (z != null) zScores.push(z)
|
||
}
|
||
if (zScores.length > 0) {
|
||
normalizedCriterionAverages[criterionId] = zScores.reduce((a, b) => a + b, 0) / zScores.length
|
||
}
|
||
}
|
||
|
||
const passRate = computePassRate(evals)
|
||
projects.push({
|
||
id: projectId,
|
||
competitionCategory: category,
|
||
avgGlobalScore,
|
||
normalizedAvgScore,
|
||
passRate,
|
||
evaluatorCount: evals.length,
|
||
criterionAverages,
|
||
normalizedCriterionAverages,
|
||
})
|
||
}
|
||
|
||
return { projects, criteriaWeights, criterionDefs, scoreWeight, passRateWeight }
|
||
}
|
||
|
||
/**
|
||
* Internal helper: fetch eligible projects for one category and execute AI ranking.
|
||
* Excluded: withdrawn projects and projects with zero submitted evaluations (locked decision).
|
||
*
|
||
* Exported so the tRPC router can call it separately when executing pre-parsed rules.
|
||
*/
|
||
export async function fetchAndRankCategory(
|
||
category: CompetitionCategory,
|
||
parsedRules: ParsedRankingRule[],
|
||
roundId: string,
|
||
prisma: PrismaClient,
|
||
userId?: string,
|
||
): Promise<RankingResult> {
|
||
const { projects, criteriaWeights, criterionDefs, scoreWeight, passRateWeight } =
|
||
await fetchCategoryProjects(category, roundId, prisma)
|
||
return executeAIRanking(
|
||
parsedRules, projects, category, criteriaWeights, criterionDefs,
|
||
userId, roundId, scoreWeight, passRateWeight,
|
||
)
|
||
}
|
||
|
||
/**
|
||
* Formula-only ranking for one category — no LLM calls.
|
||
* Computes compositeScore for each project and sorts by score descending.
|
||
*/
|
||
function formulaRankCategory(
|
||
category: CompetitionCategory,
|
||
data: CategoryProjectData,
|
||
): RankingResult {
|
||
const { projects, criteriaWeights, criterionDefs, scoreWeight, passRateWeight } = data
|
||
|
||
if (projects.length === 0) {
|
||
return { category, rankedProjects: [], parsedRules: [], totalEligible: 0 }
|
||
}
|
||
|
||
const maxEvaluatorCount = Math.max(...projects.map((p) => p.evaluatorCount))
|
||
|
||
const rankedProjects: RankedProjectEntry[] = projects
|
||
.map((p) => ({
|
||
projectId: p.id,
|
||
rank: 0,
|
||
compositeScore: computeCompositeScore(
|
||
p, maxEvaluatorCount, criteriaWeights, criterionDefs, scoreWeight, passRateWeight,
|
||
),
|
||
avgGlobalScore: p.avgGlobalScore,
|
||
normalizedAvgScore: p.normalizedAvgScore,
|
||
passRate: p.passRate,
|
||
evaluatorCount: p.evaluatorCount,
|
||
}))
|
||
.sort((a, b) => b.compositeScore - a.compositeScore)
|
||
|
||
// Assign contiguous ranks
|
||
rankedProjects.forEach((p, i) => { p.rank = i + 1 })
|
||
|
||
return { category, rankedProjects, parsedRules: [], totalEligible: projects.length }
|
||
}
|
||
|
||
/**
|
||
* Formula-only ranking: rank all projects by configurable composite score (no LLM).
|
||
* Uses scoreWeight/passRateWeight from round config + per-criterion weights.
|
||
* Returns results for both categories.
|
||
*/
|
||
export async function formulaRank(
|
||
roundId: string,
|
||
prisma: PrismaClient,
|
||
): Promise<{ startup: RankingResult; concept: RankingResult }> {
|
||
const [startupData, conceptData] = await Promise.all([
|
||
fetchCategoryProjects('STARTUP', roundId, prisma),
|
||
fetchCategoryProjects('BUSINESS_CONCEPT', roundId, prisma),
|
||
])
|
||
return {
|
||
startup: formulaRankCategory('STARTUP', startupData),
|
||
concept: formulaRankCategory('BUSINESS_CONCEPT', conceptData),
|
||
}
|
||
}
|