fix: compute z-context per-round in edition-mode rankings rollup

Previously the edition-level branch of analytics.getProjectRankings
(programId mode) pooled every juror's evaluations across every round
into a single z-normalization context. A juror's mean and stddev are
not stable across round types — quick intake screening produces a
very different grading profile than a deep evaluation round, and
mixing them yields a meaningless personal calibration.

The rollup now groups points by roundId, computes one balance context
per round, and aggregates per-project as the unweighted mean of the
per-round balanced averages. roundId mode is unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-04-27 13:14:30 +02:00
parent 7147115918
commit 97d1f2a3af
3 changed files with 180 additions and 6 deletions

View File

@@ -6,7 +6,13 @@ import { getProjectLogoUrl } from '../utils/project-logo-url'
import { aggregateVotes } from '../services/deliberation'
import { validateRoundConfig } from '@/types/competition-configs'
import type { LiveFinalConfig } from '@/types/competition-configs'
import { computeBalanceContext, computeBalancedProjectScores, type ScorePoint } from '../services/juror-balance'
import {
computeBalanceContext,
computeBalancedProjectScores,
computePerRoundBalanced,
type ScorePoint,
type RoundScopedScorePoint,
} from '../services/juror-balance'
import { generateJurorCalibration } from '../services/ai-juror-calibration'
const editionOrRoundInput = z.object({
@@ -213,24 +219,39 @@ export const analyticsRouter = router({
where: evalWhere(input, { status: 'SUBMITTED' }),
select: {
criterionScoresJson: true,
assignment: { select: { userId: true, projectId: true } },
assignment: { select: { userId: true, projectId: true, roundId: true } },
},
}),
])
// Extract a single eval-level score (mean of numeric criterion scores) per evaluation.
const points: ScorePoint[] = []
const rawPoints: RoundScopedScorePoint[] = []
for (const e of evaluations) {
const scores = e.criterionScoresJson as Record<string, unknown> | null
if (!scores) continue
const vals = Object.values(scores).filter((s): s is number => typeof s === 'number')
if (vals.length === 0) continue
const rawScore = vals.reduce((a, b) => a + b, 0) / vals.length
points.push({ projectId: e.assignment.projectId, userId: e.assignment.userId, rawScore })
rawPoints.push({
projectId: e.assignment.projectId,
userId: e.assignment.userId,
roundId: e.assignment.roundId,
rawScore,
})
}
const balanceCtx = computeBalanceContext(points)
const balancedByProject = computeBalancedProjectScores(points, balanceCtx)
// roundId mode: single-round z-context (existing behavior).
// programId mode: per-round z-contexts aggregated as the mean of per-round
// balanced averages — never pool z-contexts across rounds because a juror's
// grading profile differs by round type.
const balancedByProject: Map<string, { rawAverage: number | null; balancedAverage: number | null; count: number }> = (() => {
if (input.roundId) {
const flat: ScorePoint[] = rawPoints.map(({ projectId, userId, rawScore }) => ({ projectId, userId, rawScore }))
const ctx = computeBalanceContext(flat)
return computeBalancedProjectScores(flat, ctx)
}
return computePerRoundBalanced(rawPoints)
})()
const rankings = projects
.map((project) => {