From 6abf962fa0d2978f316aa21522699f69e17eab2b Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 18 Feb 2026 16:16:55 +0100 Subject: [PATCH] Fix AI assignment workload imbalance: enforce caps and rebalance Root cause: batches of 15 projects were processed independently - GPT didn't see assignments from previous batches, so expert jurors got assigned 18-22 projects while others got 4-5. Fixes: - Track cumulative assignments across batches (feed to each batch) - Calculate ideal target per juror and communicate to GPT - Add post-processing rebalancer that enforces hard caps and redistributes excess assignments to least-loaded jurors - Calculate sensible default max cap when not configured - Reweight prompt: workload balance 50%, expertise 35%, diversity 15% Co-Authored-By: Claude Opus 4.6 --- src/server/routers/roundAssignment.ts | 8 +- src/server/services/ai-assignment.ts | 190 +++++++++++++++++++++++--- 2 files changed, 178 insertions(+), 20 deletions(-) diff --git a/src/server/routers/roundAssignment.ts b/src/server/routers/roundAssignment.ts index ed4d3f5..ae4aa6c 100644 --- a/src/server/routers/roundAssignment.ts +++ b/src/server/routers/roundAssignment.ts @@ -134,7 +134,13 @@ export const roundAssignmentRouter = router({ // Build constraints const configJson = round.configJson as Record | null - const maxPerJuror = (configJson?.maxAssignmentsPerJuror as number) ?? undefined + const configuredMax = (configJson?.maxAssignmentsPerJuror as number) ?? undefined + + // If no explicit cap, calculate a balanced one: ceil(total_needed / juror_count) + 2 buffer + const totalNeeded = projectStates.length * input.requiredReviews + const jurorCount = round.juryGroup.members.length + const calculatedMax = Math.ceil(totalNeeded / jurorCount) + 2 + const maxPerJuror = configuredMax ?? calculatedMax const constraints = { requiredReviewsPerProject: input.requiredReviews, diff --git a/src/server/services/ai-assignment.ts b/src/server/services/ai-assignment.ts index aa39e4f..8e5728b 100644 --- a/src/server/services/ai-assignment.ts +++ b/src/server/services/ai-assignment.ts @@ -35,16 +35,21 @@ const ASSIGNMENT_BATCH_SIZE = 15 const ASSIGNMENT_SYSTEM_PROMPT = `You are an expert jury assignment optimizer for an ocean conservation competition. ## Your Role -Match jurors to projects based on expertise alignment, workload balance, geographic diversity, and coverage requirements. You have access to rich data about both jurors and projects — use ALL available information to make optimal assignments. +Match jurors to projects based on BALANCED workload distribution and expertise alignment. Even distribution is the TOP PRIORITY. ## Available Data -- **Jurors**: expertiseTags (areas of expertise), bio (background description with deeper domain knowledge), country, currentAssignmentCount, maxAssignments -- **Projects**: title, description (detailed project overview), tags (with confidence 0-1), category (e.g. STARTUP, BUSINESS_CONCEPT), oceanIssue (focus area like CORAL_REEFS, POLLUTION), country, institution, teamSize, fileTypes (submitted document types) +- **Jurors**: expertiseTags, bio, country, currentAssignmentCount, maxAssignments +- **Projects**: title, description, tags (with confidence 0-1), category, oceanIssue, country, institution, teamSize, fileTypes + +## CRITICAL RULES (Must Follow) +1. **HARD CAP**: NEVER assign a juror more than their maxAssignments. Check currentAssignmentCount + new assignments in this batch. If currentAssignmentCount >= maxAssignments, do NOT assign that juror at all. +2. **EVEN DISTRIBUTION**: Assignments must be spread as evenly as possible. Before assigning a project, always pick the juror with the FEWEST total assignments (currentAssignmentCount + assignments already given in this batch). Only deviate from this if the least-loaded juror has zero expertise relevance AND another juror with only 1-2 more assignments has strong expertise. +3. **TARGET_PER_JUROR**: A target per juror is provided. Stay within ±1 of this target for every juror. ## Matching Criteria (Weighted) -- Expertise & Domain Match (50%): How well juror tags, bio, and background align with project topics, category, ocean issue, and description. Use bio text to identify deeper domain expertise beyond explicit tags — e.g., a bio mentioning "20 years of coral research" matches coral-related projects even without explicit tags. Weight higher-confidence tags more heavily. -- Workload Balance (30%): Distribute assignments as evenly as possible; strongly prefer jurors below capacity. Never let one juror get significantly more assignments than another. -- Minimum Target (20%): Prioritize jurors who haven't reached their minimum assignment count +- Workload Balance (50%): Prefer the least-loaded juror. A juror with 2 fewer assignments than another should ALWAYS be preferred unless they have zero relevance. +- Expertise & Domain Match (35%): Tag overlap, bio background, ocean issue alignment. Use bio text for deeper matching. +- Geographic/Disciplinary Diversity (15%): Avoid same-country matches; mix expertise backgrounds per project. ## Output Format Return a JSON object: @@ -55,21 +60,16 @@ Return a JSON object: "project_id": "PROJECT_001", "confidence_score": 0.0-1.0, "expertise_match_score": 0.0-1.0, - "reasoning": "1-2 sentence justification referencing specific expertise matches" + "reasoning": "1-2 sentence justification" } ] } ## Guidelines -- Each project MUST receive the required number of reviews — ensure full coverage -- Distribute assignments as evenly as possible across all jurors -- Do not assign jurors who are at or above their capacity -- Favor geographic diversity: avoid assigning jurors from the same country as the project when possible -- Consider disciplinary diversity: mix different expertise backgrounds per project -- confidence_score reflects overall assignment quality; expertise_match_score reflects tag/expertise overlap -- A strong match: shared expertise tags + relevant bio background + available capacity -- An acceptable match: related domain/ocean issue + available capacity -- A poor match: no expertise overlap, only assigned for coverage` +- Each project MUST receive the required number of reviews +- NEVER exceed a juror's maxAssignments cap — this is a hard constraint +- Spread assignments evenly — the difference between the most-loaded and least-loaded juror should be at most 2 +- confidence_score reflects overall assignment quality; expertise_match_score reflects expertise overlap` // ─── Types ─────────────────────────────────────────────────────────────────── @@ -129,6 +129,8 @@ interface AssignmentConstraints { jurorId: string projectId: string }> + /** Ideal target assignments per juror (for balanced distribution hint) */ + _targetPerJuror?: number } export interface AssignmentProgressCallback { @@ -351,10 +353,15 @@ function buildBatchPrompt( } } + const targetStr = constraints._targetPerJuror + ? `\nTARGET_PER_JUROR: ${constraints._targetPerJuror} (aim for this many total assignments per juror, ±1)` + : '' + return `JURORS: ${JSON.stringify(jurors)} PROJECTS: ${JSON.stringify(projects)} -CONSTRAINTS: ${constraints.requiredReviewsPerProject} reviews/project, max ${constraints.maxAssignmentsPerJuror || 'unlimited'}/juror${jurorLimitsStr} +CONSTRAINTS: ${constraints.requiredReviewsPerProject} reviews/project, max ${constraints.maxAssignmentsPerJuror || 'unlimited'}/juror (HARD LIMIT — never exceed)${jurorLimitsStr}${targetStr} EXISTING: ${JSON.stringify(anonymousExisting)} +IMPORTANT: Check each juror's currentAssignmentCount + existing assignments before assigning. If at or over max, skip them entirely. Distribute evenly — pick the least-loaded juror first. Return JSON: {"assignments": [...]}` } @@ -398,6 +405,16 @@ export async function generateAIAssignments( const allSuggestions: AIAssignmentSuggestion[] = [] let totalTokens = 0 + // Calculate ideal distribution for the prompt + const totalNeededAssignments = projects.length * constraints.requiredReviewsPerProject + const maxCap = constraints.maxAssignmentsPerJuror ?? Math.ceil(totalNeededAssignments / jurors.length) + 2 + const idealPerJuror = Math.ceil(totalNeededAssignments / jurors.length) + + // Track cumulative assignments across batches (real IDs) + const cumulativeAssignments: Array<{ jurorId: string; projectId: string }> = [ + ...constraints.existingAssignments, + ] + // Process projects in batches const totalBatches = Math.ceil(anonymizedData.projects.length / ASSIGNMENT_BATCH_SIZE) @@ -408,17 +425,30 @@ export async function generateAIAssignments( console.log(`[AI Assignment] Processing batch ${currentBatch}/${totalBatches}`) + // Pass cumulative assignments so GPT knows about previous batch results + const batchConstraints: AssignmentConstraints = { + ...constraints, + maxAssignmentsPerJuror: maxCap, + existingAssignments: cumulativeAssignments, + _targetPerJuror: idealPerJuror, + } + const { suggestions, tokensUsed } = await processAssignmentBatch( openai, model, anonymizedData, batchProjects, batchMappings, - constraints, + batchConstraints, userId, entityId ) + // Add this batch's results to cumulative tracking + for (const s of suggestions) { + cumulativeAssignments.push({ jurorId: s.jurorId, projectId: s.projectId }) + } + allSuggestions.push(...suggestions) totalTokens += tokensUsed @@ -436,9 +466,12 @@ export async function generateAIAssignments( console.log(`[AI Assignment] Completed. Total suggestions: ${allSuggestions.length}, Total tokens: ${totalTokens}`) + // Post-process: enforce hard cap and rebalance + const balanced = rebalanceAssignments(allSuggestions, jurors, constraints, maxCap) + return { success: true, - suggestions: allSuggestions, + suggestions: balanced, tokensUsed: totalTokens, fallbackUsed: false, } @@ -468,6 +501,125 @@ export async function generateAIAssignments( } } +// ─── Post-Processing Rebalancer ───────────────────────────────────────────── + +/** + * Enforce hard caps and rebalance assignments from overloaded jurors. + * Moves excess assignments from over-cap jurors to under-loaded jurors + * that haven't been assigned that project yet. + */ +function rebalanceAssignments( + suggestions: AIAssignmentSuggestion[], + jurors: JurorForAssignment[], + constraints: AssignmentConstraints, + maxCap: number, +): AIAssignmentSuggestion[] { + // Build juror load tracking (existing DB assignments + new AI suggestions) + const jurorLoad = new Map() + const jurorSet = new Set(jurors.map((j) => j.id)) + + // Count existing assignments from DB + for (const ea of constraints.existingAssignments) { + if (jurorSet.has(ea.jurorId)) { + jurorLoad.set(ea.jurorId, (jurorLoad.get(ea.jurorId) || 0) + 1) + } + } + + // Count new suggestions per juror + const newLoadPerJuror = new Map() + for (const s of suggestions) { + newLoadPerJuror.set(s.jurorId, (newLoadPerJuror.get(s.jurorId) || 0) + 1) + } + + // Effective cap: per-juror personal cap or global cap + const getEffectiveCap = (jurorId: string) => { + if (constraints.jurorLimits?.[jurorId]) return constraints.jurorLimits[jurorId] + const juror = jurors.find((j) => j.id === jurorId) + return juror?.maxAssignments ?? maxCap + } + + // Calculate max new assignments allowed per juror + const maxNewForJuror = (jurorId: string) => { + const existing = jurorLoad.get(jurorId) || 0 + const cap = getEffectiveCap(jurorId) + return Math.max(0, cap - existing) + } + + // Sort suggestions by confidence (keep best matches when trimming) + const sorted = [...suggestions].sort((a, b) => b.confidenceScore - a.confidenceScore) + + // Phase 1: Accept assignments up to each juror's cap + const accepted: AIAssignmentSuggestion[] = [] + const rejected: AIAssignmentSuggestion[] = [] + const acceptedPerJuror = new Map() + const acceptedPairs = new Set() + const projectCoverage = new Map() // projectId → accepted reviewers + + for (const s of sorted) { + const currentNew = acceptedPerJuror.get(s.jurorId) || 0 + const allowed = maxNewForJuror(s.jurorId) + const pairKey = `${s.jurorId}:${s.projectId}` + + if (currentNew < allowed && !acceptedPairs.has(pairKey)) { + accepted.push(s) + acceptedPerJuror.set(s.jurorId, currentNew + 1) + acceptedPairs.add(pairKey) + projectCoverage.set(s.projectId, (projectCoverage.get(s.projectId) || 0) + 1) + } else { + rejected.push(s) + } + } + + // Phase 2: Reassign rejected items to least-loaded jurors that aren't at cap + for (const r of rejected) { + const currentCoverage = projectCoverage.get(r.projectId) || 0 + if (currentCoverage >= constraints.requiredReviewsPerProject) continue // project is covered + + // Find the least-loaded juror who can take this project + const candidates = jurors + .filter((j) => { + const pairKey = `${j.id}:${r.projectId}` + if (acceptedPairs.has(pairKey)) return false // already assigned + // Check existing DB pairs too + if (constraints.existingAssignments.some( + (ea) => ea.jurorId === j.id && ea.projectId === r.projectId + )) return false + const currentNew = acceptedPerJuror.get(j.id) || 0 + return currentNew < maxNewForJuror(j.id) + }) + .sort((a, b) => { + const aTotal = (jurorLoad.get(a.id) || 0) + (acceptedPerJuror.get(a.id) || 0) + const bTotal = (jurorLoad.get(b.id) || 0) + (acceptedPerJuror.get(b.id) || 0) + return aTotal - bTotal // least loaded first + }) + + if (candidates.length > 0) { + const picked = candidates[0] + accepted.push({ + jurorId: picked.id, + projectId: r.projectId, + confidenceScore: r.confidenceScore * 0.8, // slightly lower confidence for reassigned + expertiseMatchScore: r.expertiseMatchScore * 0.5, + reasoning: `Reassigned for workload balance (originally suggested for another juror at capacity).`, + }) + acceptedPerJuror.set(picked.id, (acceptedPerJuror.get(picked.id) || 0) + 1) + acceptedPairs.add(`${picked.id}:${r.projectId}`) + projectCoverage.set(r.projectId, (projectCoverage.get(r.projectId) || 0) + 1) + } + } + + // Log rebalancing stats + const rebalanced = accepted.length - (suggestions.length - rejected.length) + if (rejected.length > 0) { + console.log( + `[AI Assignment] Rebalanced: ${rejected.length} over-cap assignments redistributed, ` + + `${rebalanced} successfully reassigned` + ) + } + + return accepted +} + // ─── Fallback Algorithm ────────────────────────────────────────────────────── /**