3ccf9b0542
Add ability to define completely different evaluation criteria for each competition category. Admins toggle "Separate Criteria per Category" in round config, then configure criteria independently via tabbed editor. - Schema: add nullable `category` to EvaluationForm with updated constraints - Config: add `perCategoryCriteria` boolean to EvaluationConfigSchema - Helper: new `findActiveForm()` with category-aware resolution + fallback - Backend: getForm, upsertForm, getStageForm, startStage all category-aware - AI services: use project category for form lookup in summaries + ranking - Export/ranking: merge criteria from all active forms for cross-category reports - Admin UI: toggle switch + tabbed criteria editor with per-category builders - Jury UI: auto-selects correct form based on project category (invisible to juror) - Fully backwards compatible: toggle defaults OFF, existing forms unchanged Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
517 lines
18 KiB
TypeScript
517 lines
18 KiB
TypeScript
import { router, adminProcedure, withAIRateLimit } from '../trpc'
|
|
import { z } from 'zod'
|
|
import { TRPCError } from '@trpc/server'
|
|
import type { Prisma } from '@prisma/client'
|
|
import {
|
|
parseRankingCriteria,
|
|
executeAIRanking,
|
|
quickRank as aiQuickRank,
|
|
formulaRank,
|
|
fetchAndRankCategory,
|
|
type ParsedRankingRule,
|
|
} from '../services/ai-ranking'
|
|
import { logAudit } from '../utils/audit'
|
|
import type { EvaluationConfig } from '@/types/competition-configs'
|
|
|
|
// ─── Local Types ───────────────────────────────────────────────────────────────
|
|
|
|
type ReorderEvent = {
|
|
category: 'STARTUP' | 'BUSINESS_CONCEPT'
|
|
orderedProjectIds: string[]
|
|
reorderedBy: string
|
|
reorderedAt: string
|
|
}
|
|
|
|
// ─── Zod Schemas ──────────────────────────────────────────────────────────────
|
|
|
|
const ParsedRuleSchema = z.object({
|
|
step: z.number().int(),
|
|
type: z.enum(['filter', 'sort', 'limit']),
|
|
description: z.string(),
|
|
field: z.enum(['pass_rate', 'avg_score', 'evaluator_count']).nullable(),
|
|
operator: z.enum(['gte', 'lte', 'eq', 'top_n']).nullable(),
|
|
value: z.number().nullable(),
|
|
dataAvailable: z.boolean(),
|
|
})
|
|
|
|
// ─── Router ───────────────────────────────────────────────────────────────────
|
|
|
|
export const rankingRouter = router({
|
|
/**
|
|
* Parse natural-language criteria into structured rules (preview mode).
|
|
* RANK-01, RANK-02, RANK-03 — admin reviews parsed rules before executing.
|
|
*/
|
|
parseRankingCriteria: adminProcedure
|
|
.input(
|
|
z.object({
|
|
roundId: z.string(),
|
|
criteriaText: z.string().min(1).max(5000),
|
|
}),
|
|
)
|
|
.mutation(async ({ ctx, input }): Promise<ParsedRankingRule[]> => {
|
|
const rules = await parseRankingCriteria(input.criteriaText, ctx.user.id, input.roundId)
|
|
|
|
await logAudit({
|
|
prisma: ctx.prisma,
|
|
userId: ctx.user.id,
|
|
action: 'RANKING_CRITERIA_PARSED',
|
|
entityType: 'Round',
|
|
entityId: input.roundId,
|
|
detailsJson: { ruleCount: rules.length },
|
|
})
|
|
|
|
return rules
|
|
}),
|
|
|
|
/**
|
|
* Execute ranking using pre-parsed rules (confirmed mode).
|
|
* Fetches both categories in parallel, persists a RankingSnapshot.
|
|
* RANK-05, RANK-06, RANK-08.
|
|
*/
|
|
executeRanking: adminProcedure
|
|
.use(withAIRateLimit)
|
|
.input(
|
|
z.object({
|
|
roundId: z.string(),
|
|
criteriaText: z.string(),
|
|
parsedRules: z.array(ParsedRuleSchema),
|
|
}),
|
|
)
|
|
.mutation(async ({ ctx, input }) => {
|
|
// Cast to service type — validated by Zod above
|
|
const rules = input.parsedRules as ParsedRankingRule[]
|
|
|
|
// Fetch and rank both categories in parallel
|
|
const [startup, concept] = await Promise.all([
|
|
fetchAndRankCategory('STARTUP', rules, input.roundId, ctx.prisma, ctx.user.id),
|
|
fetchAndRankCategory('BUSINESS_CONCEPT', rules, input.roundId, ctx.prisma, ctx.user.id),
|
|
])
|
|
|
|
// Read criteria weights for snapshot audit trail
|
|
const round = await ctx.prisma.round.findUniqueOrThrow({
|
|
where: { id: input.roundId },
|
|
select: { configJson: true },
|
|
})
|
|
const evalConfig = (round.configJson as EvaluationConfig | null) ?? ({} as EvaluationConfig)
|
|
const criteriaWeights = evalConfig.criteriaWeights ?? undefined
|
|
|
|
// Persist snapshot — embed weights alongside rules for audit
|
|
const parsedRulesWithWeights = { rules, weights: criteriaWeights } as unknown as Prisma.InputJsonValue
|
|
const snapshot = await ctx.prisma.rankingSnapshot.create({
|
|
data: {
|
|
roundId: input.roundId,
|
|
triggeredById: ctx.user.id,
|
|
triggerType: 'MANUAL',
|
|
mode: 'CONFIRMED',
|
|
status: 'COMPLETED',
|
|
criteriaText: input.criteriaText,
|
|
parsedRulesJson: parsedRulesWithWeights,
|
|
startupRankingJson: startup.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
conceptRankingJson: concept.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
},
|
|
})
|
|
|
|
await logAudit({
|
|
prisma: ctx.prisma,
|
|
userId: ctx.user.id,
|
|
action: 'RANKING_EXECUTED',
|
|
entityType: 'RankingSnapshot',
|
|
entityId: snapshot.id,
|
|
detailsJson: {
|
|
roundId: input.roundId,
|
|
startupCount: startup.rankedProjects.length,
|
|
conceptCount: concept.rankedProjects.length,
|
|
},
|
|
})
|
|
|
|
return { snapshotId: snapshot.id, startup, concept }
|
|
}),
|
|
|
|
/**
|
|
* Quick rank: parse criteria and execute in one step (RANK-04).
|
|
* Persists a RankingSnapshot with mode=QUICK.
|
|
*/
|
|
quickRank: adminProcedure
|
|
.input(
|
|
z.object({
|
|
roundId: z.string(),
|
|
criteriaText: z.string().min(1).max(5000),
|
|
}),
|
|
)
|
|
.mutation(async ({ ctx, input }) => {
|
|
const { startup, concept, parsedRules } = await aiQuickRank(
|
|
input.criteriaText,
|
|
input.roundId,
|
|
ctx.prisma,
|
|
ctx.user.id,
|
|
)
|
|
|
|
// Persist snapshot
|
|
const snapshot = await ctx.prisma.rankingSnapshot.create({
|
|
data: {
|
|
roundId: input.roundId,
|
|
triggeredById: ctx.user.id,
|
|
triggerType: 'QUICK',
|
|
mode: 'QUICK',
|
|
status: 'COMPLETED',
|
|
criteriaText: input.criteriaText,
|
|
parsedRulesJson: parsedRules as unknown as Prisma.InputJsonValue,
|
|
startupRankingJson: startup.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
conceptRankingJson: concept.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
},
|
|
})
|
|
|
|
await logAudit({
|
|
prisma: ctx.prisma,
|
|
userId: ctx.user.id,
|
|
action: 'RANKING_QUICK_EXECUTED',
|
|
entityType: 'RankingSnapshot',
|
|
entityId: snapshot.id,
|
|
detailsJson: {
|
|
roundId: input.roundId,
|
|
startupCount: startup.rankedProjects.length,
|
|
conceptCount: concept.rankedProjects.length,
|
|
},
|
|
})
|
|
|
|
return { snapshotId: snapshot.id, startup, concept, parsedRules }
|
|
}),
|
|
|
|
/**
|
|
* List all ranking snapshots for a round (Phase 2 dashboard prep).
|
|
* Ordered by createdAt desc — newest first.
|
|
*/
|
|
listSnapshots: adminProcedure
|
|
.input(z.object({ roundId: z.string() }))
|
|
.query(async ({ ctx, input }) => {
|
|
return ctx.prisma.rankingSnapshot.findMany({
|
|
where: { roundId: input.roundId },
|
|
orderBy: { createdAt: 'desc' },
|
|
select: {
|
|
id: true,
|
|
triggeredById: true,
|
|
triggerType: true,
|
|
mode: true,
|
|
status: true,
|
|
criteriaText: true,
|
|
tokensUsed: true,
|
|
createdAt: true,
|
|
triggeredBy: {
|
|
select: { name: true },
|
|
},
|
|
},
|
|
})
|
|
}),
|
|
|
|
/**
|
|
* Get a single ranking snapshot by ID (Phase 2 dashboard prep).
|
|
* Returns full snapshot including ranking results.
|
|
*/
|
|
getSnapshot: adminProcedure
|
|
.input(z.object({ snapshotId: z.string() }))
|
|
.query(async ({ ctx, input }) => {
|
|
const snapshot = await ctx.prisma.rankingSnapshot.findUnique({
|
|
where: { id: input.snapshotId },
|
|
})
|
|
|
|
if (!snapshot) {
|
|
throw new TRPCError({
|
|
code: 'NOT_FOUND',
|
|
message: `Ranking snapshot ${input.snapshotId} not found`,
|
|
})
|
|
}
|
|
|
|
return snapshot
|
|
}),
|
|
|
|
/** Persist admin drag-reorder to RankingSnapshot.reordersJson. Append-only — never overwrites old entries. DASH-02, DASH-03. */
|
|
saveReorder: adminProcedure
|
|
.input(
|
|
z.object({
|
|
snapshotId: z.string(),
|
|
category: z.enum(['STARTUP', 'BUSINESS_CONCEPT']),
|
|
orderedProjectIds: z.array(z.string()),
|
|
}),
|
|
)
|
|
.mutation(async ({ ctx, input }) => {
|
|
const snapshot = await ctx.prisma.rankingSnapshot.findUniqueOrThrow({
|
|
where: { id: input.snapshotId },
|
|
select: { reordersJson: true },
|
|
})
|
|
|
|
const existingReorders = (snapshot.reordersJson as ReorderEvent[] | null) ?? []
|
|
|
|
const newReorder: ReorderEvent = {
|
|
category: input.category,
|
|
orderedProjectIds: input.orderedProjectIds,
|
|
reorderedBy: ctx.user.id,
|
|
reorderedAt: new Date().toISOString(),
|
|
}
|
|
|
|
await ctx.prisma.rankingSnapshot.update({
|
|
where: { id: input.snapshotId },
|
|
data: { reordersJson: [...existingReorders, newReorder] as unknown as Prisma.InputJsonValue },
|
|
})
|
|
|
|
return { ok: true }
|
|
}),
|
|
|
|
/**
|
|
* RANK-09 — Manual trigger for auto-rank (admin button on round detail page).
|
|
* Reads ranking criteria from round configJson and executes quickRank.
|
|
*/
|
|
triggerAutoRank: adminProcedure
|
|
.use(withAIRateLimit)
|
|
.input(z.object({ roundId: z.string() }))
|
|
.mutation(async ({ ctx, input }) => {
|
|
const { roundId } = input
|
|
|
|
const round = await ctx.prisma.round.findUniqueOrThrow({
|
|
where: { id: roundId },
|
|
select: { id: true, name: true, configJson: true, competition: { select: { id: true } } },
|
|
})
|
|
|
|
const config = (round.configJson as EvaluationConfig | null) ?? ({} as EvaluationConfig)
|
|
const criteriaText = config?.rankingCriteria?.trim() || null
|
|
const isFormulaMode = !criteriaText
|
|
|
|
// Create a RUNNING snapshot so all admins see the in-progress indicator
|
|
const snapshot = await ctx.prisma.rankingSnapshot.create({
|
|
data: {
|
|
roundId,
|
|
triggeredById: ctx.user.id,
|
|
triggerType: 'MANUAL',
|
|
criteriaText: criteriaText ?? '',
|
|
parsedRulesJson: {} as Prisma.InputJsonValue,
|
|
mode: isFormulaMode ? 'FORMULA' : 'QUICK',
|
|
status: 'RUNNING',
|
|
},
|
|
})
|
|
|
|
try {
|
|
let startup: { rankedProjects: unknown[] }
|
|
let concept: { rankedProjects: unknown[] }
|
|
let parsedRulesWithWeights: Prisma.InputJsonValue
|
|
|
|
if (isFormulaMode) {
|
|
// Formula-only: no LLM, pure math ranking
|
|
const result = await formulaRank(roundId, ctx.prisma)
|
|
startup = result.startup
|
|
concept = result.concept
|
|
const criteriaWeights = config.criteriaWeights ?? undefined
|
|
parsedRulesWithWeights = {
|
|
rules: [],
|
|
weights: criteriaWeights,
|
|
scoreWeight: config.scoreWeight ?? 5,
|
|
passRateWeight: config.passRateWeight ?? 5,
|
|
} as unknown as Prisma.InputJsonValue
|
|
} else {
|
|
// AI-assisted: parse criteria + rank with LLM
|
|
const result = await aiQuickRank(criteriaText, roundId, ctx.prisma, ctx.user.id)
|
|
startup = result.startup
|
|
concept = result.concept
|
|
const criteriaWeights = config.criteriaWeights ?? undefined
|
|
parsedRulesWithWeights = { rules: result.parsedRules, weights: criteriaWeights } as unknown as Prisma.InputJsonValue
|
|
}
|
|
|
|
await ctx.prisma.rankingSnapshot.update({
|
|
where: { id: snapshot.id },
|
|
data: {
|
|
status: 'COMPLETED',
|
|
parsedRulesJson: parsedRulesWithWeights,
|
|
startupRankingJson: startup.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
conceptRankingJson: concept.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
},
|
|
})
|
|
|
|
await logAudit({
|
|
prisma: ctx.prisma,
|
|
userId: ctx.user.id,
|
|
action: 'RANKING_MANUAL_TRIGGERED',
|
|
entityType: 'RankingSnapshot',
|
|
entityId: snapshot.id,
|
|
detailsJson: { roundId, mode: isFormulaMode ? 'FORMULA' : 'AI' },
|
|
ipAddress: ctx.ip,
|
|
userAgent: ctx.userAgent,
|
|
})
|
|
|
|
return { snapshotId: snapshot.id, startup, concept }
|
|
} catch (err) {
|
|
// Mark snapshot as FAILED so the indicator clears
|
|
await ctx.prisma.rankingSnapshot.update({
|
|
where: { id: snapshot.id },
|
|
data: { status: 'FAILED' },
|
|
})
|
|
throw err
|
|
}
|
|
}),
|
|
|
|
/**
|
|
* RANK-10 — Retroactive scan: finds all active/closed rounds with autoRankOnComplete
|
|
* configured but no RETROACTIVE snapshot yet, then executes ranking for each.
|
|
* Runs sequentially to avoid hammering OpenAI.
|
|
*/
|
|
retroactiveScan: adminProcedure
|
|
.input(z.object({}))
|
|
.mutation(async ({ ctx }) => {
|
|
const rounds = await ctx.prisma.round.findMany({
|
|
where: { status: { in: ['ROUND_ACTIVE', 'ROUND_CLOSED'] } },
|
|
select: { id: true, name: true, configJson: true },
|
|
})
|
|
|
|
const results: Array<{ roundId: string; triggered: boolean; reason?: string }> = []
|
|
|
|
for (const round of rounds) {
|
|
const config = (round.configJson as EvaluationConfig | null) ?? ({} as EvaluationConfig)
|
|
const autoRankEnabled = config?.autoRankOnComplete ?? false
|
|
const criteriaText = config?.rankingCriteria ?? null
|
|
|
|
if (!autoRankEnabled || !criteriaText) {
|
|
results.push({ roundId: round.id, triggered: false, reason: 'auto-rank not configured' })
|
|
continue
|
|
}
|
|
|
|
// Check if fully evaluated
|
|
const [total, completed] = await Promise.all([
|
|
ctx.prisma.assignment.count({ where: { roundId: round.id, isRequired: true } }),
|
|
ctx.prisma.assignment.count({ where: { roundId: round.id, isRequired: true, isCompleted: true } }),
|
|
])
|
|
|
|
if (total === 0 || total !== completed) {
|
|
results.push({
|
|
roundId: round.id,
|
|
triggered: false,
|
|
reason: `${completed}/${total} assignments complete`,
|
|
})
|
|
continue
|
|
}
|
|
|
|
// Check if a RETROACTIVE snapshot already exists
|
|
const existing = await ctx.prisma.rankingSnapshot.findFirst({
|
|
where: { roundId: round.id, triggerType: 'RETROACTIVE' },
|
|
select: { id: true },
|
|
})
|
|
if (existing) {
|
|
results.push({ roundId: round.id, triggered: false, reason: 'retroactive snapshot already exists' })
|
|
continue
|
|
}
|
|
|
|
// Execute ranking sequentially to avoid rate limits
|
|
try {
|
|
const result = await aiQuickRank(criteriaText, round.id, ctx.prisma, ctx.user.id)
|
|
await ctx.prisma.rankingSnapshot.create({
|
|
data: {
|
|
roundId: round.id,
|
|
triggeredById: ctx.user.id,
|
|
triggerType: 'RETROACTIVE',
|
|
criteriaText,
|
|
parsedRulesJson: result.parsedRules as unknown as Prisma.InputJsonValue,
|
|
startupRankingJson: result.startup.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
conceptRankingJson: result.concept.rankedProjects as unknown as Prisma.InputJsonValue,
|
|
mode: 'QUICK',
|
|
status: 'COMPLETED',
|
|
},
|
|
})
|
|
results.push({ roundId: round.id, triggered: true })
|
|
} catch (err) {
|
|
results.push({ roundId: round.id, triggered: false, reason: String(err) })
|
|
}
|
|
}
|
|
|
|
await logAudit({
|
|
prisma: ctx.prisma,
|
|
userId: ctx.user.id,
|
|
action: 'RANKING_RETROACTIVE_SCAN',
|
|
entityType: 'Round',
|
|
detailsJson: { results },
|
|
ipAddress: ctx.ip,
|
|
userAgent: ctx.userAgent,
|
|
})
|
|
|
|
return {
|
|
results,
|
|
total: results.length,
|
|
triggered: results.filter((r) => r.triggered).length,
|
|
}
|
|
}),
|
|
|
|
/**
|
|
* Get per-project evaluation scores for a round.
|
|
* Returns a map of projectId → array of { jurorName, globalScore, binaryDecision }.
|
|
* Used by the ranking dashboard to show individual juror scores inline.
|
|
*/
|
|
roundEvaluationScores: adminProcedure
|
|
.input(z.object({ roundId: z.string() }))
|
|
.query(async ({ ctx, input }) => {
|
|
// Find the boolean criterion ID across all active forms (shared + category-specific)
|
|
const activeForms = await ctx.prisma.evaluationForm.findMany({
|
|
where: { roundId: input.roundId, isActive: true },
|
|
select: { criteriaJson: true },
|
|
})
|
|
let boolCriterionId: string | null = null
|
|
for (const f of activeForms) {
|
|
const fc = (f.criteriaJson as Array<{
|
|
id: string; label: string; type?: string
|
|
}> | null) ?? []
|
|
const found = fc.find(
|
|
(c) => c.type === 'boolean' && c.label?.toLowerCase().includes('move to the next stage'),
|
|
)
|
|
if (found) {
|
|
boolCriterionId = found.id
|
|
break
|
|
}
|
|
}
|
|
|
|
const assignments = await ctx.prisma.assignment.findMany({
|
|
where: {
|
|
roundId: input.roundId,
|
|
isRequired: true,
|
|
evaluation: { status: 'SUBMITTED' },
|
|
},
|
|
select: {
|
|
projectId: true,
|
|
user: { select: { name: true, email: true } },
|
|
evaluation: {
|
|
select: {
|
|
globalScore: true,
|
|
binaryDecision: true,
|
|
criterionScoresJson: true,
|
|
},
|
|
},
|
|
},
|
|
})
|
|
|
|
const byProject: Record<string, Array<{
|
|
jurorName: string
|
|
globalScore: number | null
|
|
decision: boolean | null
|
|
}>> = {}
|
|
|
|
for (const a of assignments) {
|
|
if (!a.evaluation) continue
|
|
const list = byProject[a.projectId] ?? []
|
|
|
|
// Resolve binary decision: column first, then criterion fallback
|
|
let decision = a.evaluation.binaryDecision
|
|
if (decision == null && boolCriterionId) {
|
|
const scores = a.evaluation.criterionScoresJson as Record<string, unknown> | null
|
|
if (scores) {
|
|
const val = scores[boolCriterionId]
|
|
if (typeof val === 'boolean') decision = val
|
|
else if (val === 'true') decision = true
|
|
else if (val === 'false') decision = false
|
|
}
|
|
}
|
|
|
|
list.push({
|
|
jurorName: a.user.name ?? a.user.email ?? 'Unknown',
|
|
globalScore: a.evaluation.globalScore,
|
|
decision,
|
|
})
|
|
byProject[a.projectId] = list
|
|
}
|
|
|
|
return byProject
|
|
}),
|
|
})
|