feat: formula-based ranking with optional AI, configurable score/pass-rate weights

Add scoreWeight and passRateWeight (0-10) to evaluation config for configurable composite score formula. When ranking criteria text is empty, triggerAutoRank uses pure formula ranking (no LLM calls). When criteria text is present, AI-assisted ranking runs as before. - Add FORMULA to RankingMode enum with migration - Extract fetchCategoryProjects helper, add formulaRank service - Update computeCompositeScore to accept configurable weights - Add score/pass-rate weight sliders to ranking dashboard UI - Mode-aware button labels (Calculator/formula vs Sparkles/AI) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-02 20:24:17 +01:00
parent ac86e025e2
commit cb688ba3e6
6 changed files with 226 additions and 42 deletions
--- a/prisma/migrations/20260302100000_add_formula_ranking_mode/migration.sql
+++ b/prisma/migrations/20260302100000_add_formula_ranking_mode/migration.sql
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "RankingMode" ADD VALUE 'FORMULA';
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -1425,6 +1425,7 @@ enum RankingMode {
  PREVIEW   // Parsed rules shown to admin (not yet applied)
  CONFIRMED // Admin confirmed rules, ranking applied
  QUICK     // Quick-rank: parse + apply without preview
+  FORMULA   // Formula-only: no LLM, pure math ranking
 }

 enum RankingSnapshotStatus {
--- a/src/components/admin/round/ranking-dashboard.tsx
+++ b/src/components/admin/round/ranking-dashboard.tsx
@@ -53,8 +53,10 @@ import {
 import {
  GripVertical,
  BarChart3,
+  Calculator,
  Loader2,
  RefreshCw,
+  Sparkles,
  Trophy,
  ExternalLink,
  ChevronDown,
@@ -263,6 +265,8 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
  const [weightsOpen, setWeightsOpen] = useState(false)
  const [localWeights, setLocalWeights] = useState<Record<string, number>>({})
  const [localCriteriaText, setLocalCriteriaText] = useState<string>('')
+  const [localScoreWeight, setLocalScoreWeight] = useState(5)
+  const [localPassRateWeight, setLocalPassRateWeight] = useState(5)
  const weightsInitialized = useRef(false)

  // ─── Sensors ──────────────────────────────────────────────────────────────
@@ -476,6 +480,8 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
      const saved = (cfg.criteriaWeights ?? {}) as Record<string, number>
      setLocalWeights(saved)
      setLocalCriteriaText((cfg.rankingCriteria as string) ?? '')
+      setLocalScoreWeight((cfg.scoreWeight as number) ?? 5)
+      setLocalPassRateWeight((cfg.passRateWeight as number) ?? 5)
      weightsInitialized.current = true
    }
  }, [roundData])
@@ -486,10 +492,19 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
    const cfg = roundData.configJson as Record<string, unknown>
    updateRoundMutation.mutate({
      id: roundId,
-      configJson: { ...cfg, criteriaWeights: localWeights, rankingCriteria: localCriteriaText },
+      configJson: {
+        ...cfg,
+        criteriaWeights: localWeights,
+        rankingCriteria: localCriteriaText,
+        scoreWeight: localScoreWeight,
+        passRateWeight: localPassRateWeight,
+      },
    })
  }

+  // Derive ranking mode from criteria text
+  const isFormulaMode = !localCriteriaText.trim()
+
  // ─── sync advance dialog defaults from config ────────────────────────────
  useEffect(() => {
    if (evalConfig) {
@@ -621,7 +636,7 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
              <>
                <Loader2 className="h-10 w-10 text-blue-500 animate-spin" />
                <div>
-                  <p className="font-medium">AI ranking in progress&hellip;</p>
+                  <p className="font-medium">Ranking in progress&hellip;</p>
                  <p className="mt-1 text-sm text-muted-foreground">
                    This may take a minute. You can continue working — results will appear automatically.
                  </p>
@@ -643,8 +658,12 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
                  onClick={() => triggerRankMutation.mutate({ roundId })}
                  disabled={triggerRankMutation.isPending}
                >
-                  <RefreshCw className="mr-2 h-4 w-4" />
-                  Run Ranking Now
+                  {isFormulaMode ? (
+                    <Calculator className="mr-2 h-4 w-4" />
+                  ) : (
+                    <Sparkles className="mr-2 h-4 w-4" />
+                  )}
+                  {isFormulaMode ? 'Run Ranking Now' : 'Run AI Ranking Now'}
                </Button>
              </>
            )}
@@ -714,10 +733,15 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
                    <Loader2 className="mr-2 h-4 w-4 animate-spin" />
                    Ranking&hellip;
                  </>
+                ) : isFormulaMode ? (
+                  <>
+                    <Calculator className="mr-2 h-4 w-4" />
+                    Run Ranking
+                  </>
                ) : (
                  <>
-                    <RefreshCw className="mr-2 h-4 w-4" />
-                    Run Ranking
+                    <Sparkles className="mr-2 h-4 w-4" />
+                    Run AI Ranking
                  </>
                )}
              </Button>
@@ -754,11 +778,48 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
            </CollapsibleTrigger>
            <CollapsibleContent>
              <CardContent className="space-y-5 pt-0">
-                {/* Ranking criteria text */}
+                {/* Score vs Pass Rate weights */}
+                <div className="space-y-3">
+                  <div>
+                    <Label>Formula Weights</Label>
+                    <p className="text-xs text-muted-foreground">
+                      Control the balance between evaluation scores and yes/no pass rate in the composite ranking
+                    </p>
+                  </div>
+                  <div className="space-y-3">
+                    <div className="flex items-center gap-4">
+                      <span className="text-sm w-40 flex-shrink-0">Score Weight</span>
+                      <Slider
+                        min={0}
+                        max={10}
+                        step={1}
+                        value={[localScoreWeight]}
+                        onValueChange={([v]) => setLocalScoreWeight(v)}
+                        className="flex-1"
+                      />
+                      <span className="text-sm font-mono w-6 text-right">{localScoreWeight}</span>
+                    </div>
+                    <div className="flex items-center gap-4">
+                      <span className="text-sm w-40 flex-shrink-0">Pass Rate Weight</span>
+                      <Slider
+                        min={0}
+                        max={10}
+                        step={1}
+                        value={[localPassRateWeight]}
+                        onValueChange={([v]) => setLocalPassRateWeight(v)}
+                        className="flex-1"
+                      />
+                      <span className="text-sm font-mono w-6 text-right">{localPassRateWeight}</span>
+                    </div>
+                  </div>
+                </div>
+
+                {/* Ranking criteria text (optional — triggers AI mode) */}
                <div className="space-y-2">
-                  <Label htmlFor="rankingCriteria">Ranking Criteria (natural language)</Label>
+                  <Label htmlFor="rankingCriteria">AI Ranking Criteria (optional)</Label>
                  <p className="text-xs text-muted-foreground">
-                    Describe how projects should be ranked. The AI will parse this into rules.
+                    Optional: describe special ranking criteria for AI-assisted ranking.
+                    Leave empty for formula-based ranking (faster, no AI cost).
                  </p>
                  <Textarea
                    id="rankingCriteria"
@@ -768,6 +829,15 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
                    onChange={(e) => setLocalCriteriaText(e.target.value)}
                    className="resize-y"
                  />
+                  {isFormulaMode ? (
+                    <p className="text-xs text-emerald-600 flex items-center gap-1">
+                      <Calculator className="h-3 w-3" /> Formula mode — ranking uses weights only, no AI calls
+                    </p>
+                  ) : (
+                    <p className="text-xs text-amber-600 flex items-center gap-1">
+                      <Sparkles className="h-3 w-3" /> AI mode — criteria will be parsed and used for ranking (uses API credits)
+                    </p>
+                  )}
                </div>

                {/* Per-criterion weights */}
@@ -823,7 +893,7 @@ export function RankingDashboard({ competitionId: _competitionId, roundId }: Ran
              <Loader2 className="h-5 w-5 animate-spin text-blue-600 dark:text-blue-400 flex-shrink-0" />
              <div className="flex-1 min-w-0">
                <p className="text-sm font-medium text-blue-900 dark:text-blue-200">
-                  AI ranking in progress&hellip;
+                  Ranking in progress&hellip;
                </p>
                <p className="text-xs text-blue-700 dark:text-blue-400">
                  This may take a minute. You can continue working — results will appear automatically.
--- a/src/server/routers/ranking.ts
+++ b/src/server/routers/ranking.ts
@@ -6,6 +6,7 @@ import {
  parseRankingCriteria,
  executeAIRanking,
  quickRank as aiQuickRank,
+  formulaRank,
  fetchAndRankCategory,
  type ParsedRankingRule,
 } from '../services/ai-ranking'
@@ -269,14 +270,8 @@ export const rankingRouter = router({
      })

      const config = (round.configJson as EvaluationConfig | null) ?? ({} as EvaluationConfig)
-      const criteriaText = config?.rankingCriteria ?? null
-
-      if (!criteriaText) {
-        throw new TRPCError({
-          code: 'BAD_REQUEST',
-          message: 'No ranking criteria configured for this round. Add criteria in round settings first.',
-        })
-      }
+      const criteriaText = config?.rankingCriteria?.trim() || null
+      const isFormulaMode = !criteriaText

      // Create a RUNNING snapshot so all admins see the in-progress indicator
      const snapshot = await ctx.prisma.rankingSnapshot.create({
@@ -284,26 +279,46 @@ export const rankingRouter = router({
          roundId,
          triggeredById: ctx.user.id,
          triggerType: 'MANUAL',
-          criteriaText,
+          criteriaText: criteriaText ?? '',
          parsedRulesJson: {} as Prisma.InputJsonValue,
-          mode: 'QUICK',
+          mode: isFormulaMode ? 'FORMULA' : 'QUICK',
          status: 'RUNNING',
        },
      })

      try {
-        const result = await aiQuickRank(criteriaText, roundId, ctx.prisma, ctx.user.id)
+        let startup: { rankedProjects: unknown[] }
+        let concept: { rankedProjects: unknown[] }
+        let parsedRulesWithWeights: Prisma.InputJsonValue
+
+        if (isFormulaMode) {
+          // Formula-only: no LLM, pure math ranking
+          const result = await formulaRank(roundId, ctx.prisma)
+          startup = result.startup
+          concept = result.concept
+          const criteriaWeights = config.criteriaWeights ?? undefined
+          parsedRulesWithWeights = {
+            rules: [],
+            weights: criteriaWeights,
+            scoreWeight: config.scoreWeight ?? 5,
+            passRateWeight: config.passRateWeight ?? 5,
+          } as unknown as Prisma.InputJsonValue
+        } else {
+          // AI-assisted: parse criteria + rank with LLM
+          const result = await aiQuickRank(criteriaText, roundId, ctx.prisma, ctx.user.id)
+          startup = result.startup
+          concept = result.concept
+          const criteriaWeights = config.criteriaWeights ?? undefined
+          parsedRulesWithWeights = { rules: result.parsedRules, weights: criteriaWeights } as unknown as Prisma.InputJsonValue
+        }

-        // Embed weights alongside rules for audit
-        const criteriaWeights = config.criteriaWeights ?? undefined
-        const parsedRulesWithWeights = { rules: result.parsedRules, weights: criteriaWeights } as unknown as Prisma.InputJsonValue
        await ctx.prisma.rankingSnapshot.update({
          where: { id: snapshot.id },
          data: {
            status: 'COMPLETED',
            parsedRulesJson: parsedRulesWithWeights,
-            startupRankingJson: result.startup.rankedProjects as unknown as Prisma.InputJsonValue,
-            conceptRankingJson: result.concept.rankedProjects as unknown as Prisma.InputJsonValue,
+            startupRankingJson: startup.rankedProjects as unknown as Prisma.InputJsonValue,
+            conceptRankingJson: concept.rankedProjects as unknown as Prisma.InputJsonValue,
          },
        })

@@ -313,12 +328,12 @@ export const rankingRouter = router({
          action: 'RANKING_MANUAL_TRIGGERED',
          entityType: 'RankingSnapshot',
          entityId: snapshot.id,
-          detailsJson: { roundId },
+          detailsJson: { roundId, mode: isFormulaMode ? 'FORMULA' : 'AI' },
          ipAddress: ctx.ip,
          userAgent: ctx.userAgent,
        })

-        return { snapshotId: snapshot.id, startup: result.startup, concept: result.concept }
+        return { snapshotId: snapshot.id, startup, concept }
      } catch (err) {
        // Mark snapshot as FAILED so the indicator clears
        await ctx.prisma.rankingSnapshot.update({
--- a/src/server/services/ai-ranking.ts
+++ b/src/server/services/ai-ranking.ts
@@ -172,6 +172,8 @@ function computeCompositeScore(
  maxEvaluatorCount: number,
  criteriaWeights: Record<string, number> | undefined,
  criterionDefs: CriterionDef[],
+  scoreWeight = 5,
+  passRateWeight = 5,
 ): number {
  let scoreComponent: number

@@ -201,7 +203,12 @@ function computeCompositeScore(
    scoreComponent = avg != null ? (avg - 1) / 9 : 0.5
  }

-  const composite = scoreComponent * 0.5 + project.passRate * 0.5
+  // Configurable score vs pass-rate weighting
+  const totalW = scoreWeight + passRateWeight
+  const sW = totalW > 0 ? scoreWeight / totalW : 0.5
+  const pW = totalW > 0 ? passRateWeight / totalW : 0.5
+  const composite = scoreComponent * sW + project.passRate * pW
+
  // Tiebreak: tiny bonus for more evaluators (won't change rank unless composite is equal)
  const tiebreakBonus = maxEvaluatorCount > 0
    ? (project.evaluatorCount / maxEvaluatorCount) * 0.0001
@@ -432,6 +439,8 @@ export async function executeAIRanking(
  criterionDefs: CriterionDef[],
  userId?: string,
  entityId?: string,
+  scoreWeight = 5,
+  passRateWeight = 5,
 ): Promise<RankingResult> {
  if (projects.length === 0) {
    return { category, rankedProjects: [], parsedRules, totalEligible: 0 }
@@ -531,7 +540,7 @@ export async function executeAIRanking(
      return {
        projectId: realId,
        rank: entry.rank,
-        compositeScore: computeCompositeScore(proj, maxEvaluatorCount, criteriaWeights, criterionDefs),
+        compositeScore: computeCompositeScore(proj, maxEvaluatorCount, criteriaWeights, criterionDefs, scoreWeight, passRateWeight),
        avgGlobalScore: proj.avgGlobalScore,
        normalizedAvgScore: proj.normalizedAvgScore,
        passRate: proj.passRate,
@@ -595,22 +604,27 @@ export async function quickRank(
  return { startup, concept, parsedRules }
 }

+// Result of fetchCategoryProjects — shared data for both AI and formula ranking
+interface CategoryProjectData {
+  projects: ProjectForRanking[]
+  criteriaWeights: Record<string, number> | undefined
+  criterionDefs: CriterionDef[]
+  scoreWeight: number
+  passRateWeight: number
+}
+
 /**
- * Internal helper: fetch eligible projects for one category and execute ranking.
- * Excluded: withdrawn projects and projects with zero submitted evaluations (locked decision).
+ * Shared data-gathering helper: fetch eligible projects for one category.
+ * Handles: round config + eval form loading, z-score normalization,
+ * per-criterion averages, pass rates, etc.
 *
- * Fetches evaluation form criteria, computes per-criterion averages, applies z-score
- * normalization to correct for juror bias, and passes weighted criteria to the AI.
- *
- * Exported so the tRPC router can call it separately when executing pre-parsed rules.
+ * Used by both `fetchAndRankCategory` (AI path) and `formulaRankCategory` (formula path).
 */
-export async function fetchAndRankCategory(
+async function fetchCategoryProjects(
  category: CompetitionCategory,
-  parsedRules: ParsedRankingRule[],
  roundId: string,
  prisma: PrismaClient,
-  userId?: string,
-): Promise<RankingResult> {
+): Promise<CategoryProjectData> {
  // Fetch the round config and evaluation form in parallel
  const [round, evalForm] = await Promise.all([
    prisma.round.findUniqueOrThrow({
@@ -625,9 +639,11 @@ export async function fetchAndRankCategory(

  const roundConfig = round.configJson as Record<string, unknown> | null

-  // Parse evaluation config for criteria weights
+  // Parse evaluation config for criteria weights and formula weights
  const evalConfig = roundConfig as EvaluationConfig | null
  const criteriaWeights = evalConfig?.criteriaWeights ?? undefined
+  const scoreWeight = evalConfig?.scoreWeight ?? 5
+  const passRateWeight = evalConfig?.passRateWeight ?? 5

  // Parse criterion definitions from the evaluation form
  const criterionDefs: CriterionDef[] = evalForm?.criteriaJson
@@ -770,5 +786,81 @@ export async function fetchAndRankCategory(
    })
  }

-  return executeAIRanking(parsedRules, projects, category, criteriaWeights, criterionDefs, userId, roundId)
+  return { projects, criteriaWeights, criterionDefs, scoreWeight, passRateWeight }
+}
+
+/**
+ * Internal helper: fetch eligible projects for one category and execute AI ranking.
+ * Excluded: withdrawn projects and projects with zero submitted evaluations (locked decision).
+ *
+ * Exported so the tRPC router can call it separately when executing pre-parsed rules.
+ */
+export async function fetchAndRankCategory(
+  category: CompetitionCategory,
+  parsedRules: ParsedRankingRule[],
+  roundId: string,
+  prisma: PrismaClient,
+  userId?: string,
+): Promise<RankingResult> {
+  const { projects, criteriaWeights, criterionDefs, scoreWeight, passRateWeight } =
+    await fetchCategoryProjects(category, roundId, prisma)
+  return executeAIRanking(
+    parsedRules, projects, category, criteriaWeights, criterionDefs,
+    userId, roundId, scoreWeight, passRateWeight,
+  )
+}
+
+/**
+ * Formula-only ranking for one category — no LLM calls.
+ * Computes compositeScore for each project and sorts by score descending.
+ */
+function formulaRankCategory(
+  category: CompetitionCategory,
+  data: CategoryProjectData,
+): RankingResult {
+  const { projects, criteriaWeights, criterionDefs, scoreWeight, passRateWeight } = data
+
+  if (projects.length === 0) {
+    return { category, rankedProjects: [], parsedRules: [], totalEligible: 0 }
+  }
+
+  const maxEvaluatorCount = Math.max(...projects.map((p) => p.evaluatorCount))
+
+  const rankedProjects: RankedProjectEntry[] = projects
+    .map((p) => ({
+      projectId: p.id,
+      rank: 0,
+      compositeScore: computeCompositeScore(
+        p, maxEvaluatorCount, criteriaWeights, criterionDefs, scoreWeight, passRateWeight,
+      ),
+      avgGlobalScore: p.avgGlobalScore,
+      normalizedAvgScore: p.normalizedAvgScore,
+      passRate: p.passRate,
+      evaluatorCount: p.evaluatorCount,
+    }))
+    .sort((a, b) => b.compositeScore - a.compositeScore)
+
+  // Assign contiguous ranks
+  rankedProjects.forEach((p, i) => { p.rank = i + 1 })
+
+  return { category, rankedProjects, parsedRules: [], totalEligible: projects.length }
+}
+
+/**
+ * Formula-only ranking: rank all projects by configurable composite score (no LLM).
+ * Uses scoreWeight/passRateWeight from round config + per-criterion weights.
+ * Returns results for both categories.
+ */
+export async function formulaRank(
+  roundId: string,
+  prisma: PrismaClient,
+): Promise<{ startup: RankingResult; concept: RankingResult }> {
+  const [startupData, conceptData] = await Promise.all([
+    fetchCategoryProjects('STARTUP', roundId, prisma),
+    fetchCategoryProjects('BUSINESS_CONCEPT', roundId, prisma),
+  ])
+  return {
+    startup: formulaRankCategory('STARTUP', startupData),
+    concept: formulaRankCategory('BUSINESS_CONCEPT', conceptData),
+  }
 }
--- a/src/types/competition-configs.ts
+++ b/src/types/competition-configs.ts
@@ -147,6 +147,10 @@ export const EvaluationConfigSchema = z.object({

  // Ranking (Phase 2) — per-criterion weights for AI ranking
  criteriaWeights: z.record(z.string(), z.number().min(0).max(10)).optional(),
+
+  // Ranking (Phase 3) — composite score formula weights
+  scoreWeight: z.number().min(0).max(10).default(5),
+  passRateWeight: z.number().min(0).max(10).default(5),
 })

 export type EvaluationConfig = z.infer<typeof EvaluationConfigSchema>