From 998ffe3af8e22c5d8f00df17c224f84ddf8b1b3d Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 18 Feb 2026 16:48:06 +0100 Subject: [PATCH] Fix AI assignment: generate multiple reviewers per project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problems: - GPT only generated 1 reviewer per project despite N being required - maxTokens (4000) too small for N×projects assignment objects - No fallback when GPT under-assigned Fixes: - System prompt now explicitly explains multiple reviewers per project with concrete example showing 3 different juror_ids per project - User prompt includes REVIEWS_PER_PROJECT, EXPECTED_OUTPUT_SIZE - maxTokens dynamically calculated: expectedAssignments × 200 + 500 - Reduced batch size from 15 to 10 (fewer projects per GPT call) - Added fillCoverageGaps() post-processor: algorithmically assigns least-loaded jurors to any project below required coverage Co-Authored-By: Claude Opus 4.6 --- src/server/services/ai-assignment.ts | 147 ++++++++++++++++++++++----- 1 file changed, 124 insertions(+), 23 deletions(-) diff --git a/src/server/services/ai-assignment.ts b/src/server/services/ai-assignment.ts index 8e5728b..a027b51 100644 --- a/src/server/services/ai-assignment.ts +++ b/src/server/services/ai-assignment.ts @@ -29,7 +29,7 @@ import { // ─── Constants ─────────────────────────────────────────────────────────────── -const ASSIGNMENT_BATCH_SIZE = 15 +const ASSIGNMENT_BATCH_SIZE = 10 // Structured system prompt for assignment const ASSIGNMENT_SYSTEM_PROMPT = `You are an expert jury assignment optimizer for an ocean conservation competition. @@ -42,34 +42,29 @@ Match jurors to projects based on BALANCED workload distribution and expertise a - **Projects**: title, description, tags (with confidence 0-1), category, oceanIssue, country, institution, teamSize, fileTypes ## CRITICAL RULES (Must Follow) -1. **HARD CAP**: NEVER assign a juror more than their maxAssignments. Check currentAssignmentCount + new assignments in this batch. If currentAssignmentCount >= maxAssignments, do NOT assign that juror at all. -2. **EVEN DISTRIBUTION**: Assignments must be spread as evenly as possible. Before assigning a project, always pick the juror with the FEWEST total assignments (currentAssignmentCount + assignments already given in this batch). Only deviate from this if the least-loaded juror has zero expertise relevance AND another juror with only 1-2 more assignments has strong expertise. -3. **TARGET_PER_JUROR**: A target per juror is provided. Stay within ±1 of this target for every juror. +1. **MULTIPLE REVIEWERS PER PROJECT**: Each project MUST be assigned to EXACTLY the number of DIFFERENT jurors specified in REVIEWS_PER_PROJECT. For example, if REVIEWS_PER_PROJECT is 3, every project needs 3 separate assignment objects with 3 different juror_ids. This is the most important rule. +2. **HARD CAP**: NEVER assign a juror more than their maxAssignments. Check currentAssignmentCount + new assignments in this batch. If a juror is at capacity, skip them. +3. **EVEN DISTRIBUTION**: Spread assignments evenly. Always prefer the juror with the FEWEST total assignments (currentAssignmentCount + assignments in this batch). Only deviate if the least-loaded juror has zero relevance and another with 1-2 more has strong expertise. ## Matching Criteria (Weighted) -- Workload Balance (50%): Prefer the least-loaded juror. A juror with 2 fewer assignments than another should ALWAYS be preferred unless they have zero relevance. -- Expertise & Domain Match (35%): Tag overlap, bio background, ocean issue alignment. Use bio text for deeper matching. -- Geographic/Disciplinary Diversity (15%): Avoid same-country matches; mix expertise backgrounds per project. +- Workload Balance (50%): Prefer least-loaded jurors. +- Expertise Match (35%): Tag overlap, bio background, ocean issue alignment. +- Diversity (15%): Avoid same-country; mix expertise per project. ## Output Format -Return a JSON object: +Return a JSON object. For N reviews/project, each project appears N times with DIFFERENT juror_ids: { "assignments": [ - { - "juror_id": "JUROR_001", - "project_id": "PROJECT_001", - "confidence_score": 0.0-1.0, - "expertise_match_score": 0.0-1.0, - "reasoning": "1-2 sentence justification" - } + { "juror_id": "JUROR_001", "project_id": "PROJECT_001", "confidence_score": 0.85, "expertise_match_score": 0.7, "reasoning": "justification" }, + { "juror_id": "JUROR_003", "project_id": "PROJECT_001", "confidence_score": 0.72, "expertise_match_score": 0.5, "reasoning": "justification" }, + { "juror_id": "JUROR_005", "project_id": "PROJECT_001", "confidence_score": 0.65, "expertise_match_score": 0.3, "reasoning": "justification" } ] } ## Guidelines -- Each project MUST receive the required number of reviews -- NEVER exceed a juror's maxAssignments cap — this is a hard constraint -- Spread assignments evenly — the difference between the most-loaded and least-loaded juror should be at most 2 -- confidence_score reflects overall assignment quality; expertise_match_score reflects expertise overlap` +- Total assignments in output should be approximately: number_of_projects × REVIEWS_PER_PROJECT +- NEVER exceed a juror's maxAssignments cap +- Spread evenly — max-loaded minus min-loaded juror should differ by at most 2` // ─── Types ─────────────────────────────────────────────────────────────────── @@ -177,6 +172,11 @@ async function processAssignmentBatch( let response: Awaited> try { + // Calculate maxTokens based on expected assignments + // ~150 tokens per assignment JSON object + const expectedAssignments = batchProjects.length * constraints.requiredReviewsPerProject + const estimatedTokens = Math.max(4000, expectedAssignments * 200 + 500) + const params = buildCompletionParams(model, { messages: [ { role: 'system', content: ASSIGNMENT_SYSTEM_PROMPT }, @@ -184,7 +184,7 @@ async function processAssignmentBatch( ], jsonMode: true, temperature: 0.1, - maxTokens: 4000, + maxTokens: estimatedTokens, }) try { @@ -357,11 +357,15 @@ function buildBatchPrompt( ? `\nTARGET_PER_JUROR: ${constraints._targetPerJuror} (aim for this many total assignments per juror, ±1)` : '' + const expectedTotal = projects.length * constraints.requiredReviewsPerProject + return `JURORS: ${JSON.stringify(jurors)} PROJECTS: ${JSON.stringify(projects)} -CONSTRAINTS: ${constraints.requiredReviewsPerProject} reviews/project, max ${constraints.maxAssignmentsPerJuror || 'unlimited'}/juror (HARD LIMIT — never exceed)${jurorLimitsStr}${targetStr} +REVIEWS_PER_PROJECT: ${constraints.requiredReviewsPerProject} (each project MUST get exactly ${constraints.requiredReviewsPerProject} different jurors) +MAX_PER_JUROR: ${constraints.maxAssignmentsPerJuror || 'unlimited'} (HARD LIMIT — never exceed)${jurorLimitsStr}${targetStr} EXISTING: ${JSON.stringify(anonymousExisting)} -IMPORTANT: Check each juror's currentAssignmentCount + existing assignments before assigning. If at or over max, skip them entirely. Distribute evenly — pick the least-loaded juror first. +EXPECTED_OUTPUT_SIZE: approximately ${expectedTotal} assignment objects (${projects.length} projects × ${constraints.requiredReviewsPerProject} reviewers each) +IMPORTANT: Every project must appear ${constraints.requiredReviewsPerProject} times with ${constraints.requiredReviewsPerProject} DIFFERENT juror_ids. Pick the least-loaded jurors first. Never exceed a juror's max. Return JSON: {"assignments": [...]}` } @@ -469,9 +473,14 @@ export async function generateAIAssignments( // Post-process: enforce hard cap and rebalance const balanced = rebalanceAssignments(allSuggestions, jurors, constraints, maxCap) + // Fill coverage gaps: if any project has fewer than requiredReviewsPerProject, use fallback + const gapFilled = fillCoverageGaps(balanced, jurors, projects, constraints, maxCap) + + console.log(`[AI Assignment] After gap-fill: ${gapFilled.length} total (${gapFilled.length - balanced.length} added for coverage)`) + return { success: true, - suggestions: balanced, + suggestions: gapFilled, tokensUsed: totalTokens, fallbackUsed: false, } @@ -620,6 +629,98 @@ function rebalanceAssignments( return accepted } +/** + * Fill coverage gaps — ensure every project has requiredReviewsPerProject assignments. + * Uses a simple least-loaded-juror algorithm to fill missing slots. + */ +function fillCoverageGaps( + suggestions: AIAssignmentSuggestion[], + jurors: JurorForAssignment[], + projects: ProjectForAssignment[], + constraints: AssignmentConstraints, + maxCap: number, +): AIAssignmentSuggestion[] { + const result = [...suggestions] + + // Track current state + const assignedPairs = new Set() + const jurorLoad = new Map() + const projectCoverage = new Map() + + // Count existing DB assignments + for (const ea of constraints.existingAssignments) { + assignedPairs.add(`${ea.jurorId}:${ea.projectId}`) + jurorLoad.set(ea.jurorId, (jurorLoad.get(ea.jurorId) || 0) + 1) + projectCoverage.set(ea.projectId, (projectCoverage.get(ea.projectId) || 0) + 1) + } + + // Count new AI suggestions + for (const s of suggestions) { + assignedPairs.add(`${s.jurorId}:${s.projectId}`) + jurorLoad.set(s.jurorId, (jurorLoad.get(s.jurorId) || 0) + 1) + projectCoverage.set(s.projectId, (projectCoverage.get(s.projectId) || 0) + 1) + } + + const getEffectiveCap = (jurorId: string) => { + if (constraints.jurorLimits?.[jurorId]) return constraints.jurorLimits[jurorId] + const juror = jurors.find((j) => j.id === jurorId) + return juror?.maxAssignments ?? maxCap + } + + let gapsFilled = 0 + + // For each project, check if it needs more reviewers + for (const project of projects) { + const current = projectCoverage.get(project.id) || 0 + const needed = constraints.requiredReviewsPerProject - current + + if (needed <= 0) continue + + // Find available jurors sorted by load (least loaded first) + const candidates = jurors + .filter((j) => { + const pairKey = `${j.id}:${project.id}` + if (assignedPairs.has(pairKey)) return false + const load = jurorLoad.get(j.id) || 0 + return load < getEffectiveCap(j.id) + }) + .sort((a, b) => { + const aLoad = jurorLoad.get(a.id) || 0 + const bLoad = jurorLoad.get(b.id) || 0 + return aLoad - bLoad + }) + + for (let i = 0; i < Math.min(needed, candidates.length); i++) { + const juror = candidates[i] + const expertiseScore = calculateExpertiseScore( + juror.expertiseTags, + project.tags, + project.tagConfidences, + ) + + result.push({ + jurorId: juror.id, + projectId: project.id, + confidenceScore: expertiseScore * 0.7, // slightly lower confidence for gap-fill + expertiseMatchScore: expertiseScore, + reasoning: generateFallbackReasoning(juror.expertiseTags, project.tags, expertiseScore) + + ' (Added to meet coverage requirement)', + }) + + assignedPairs.add(`${juror.id}:${project.id}`) + jurorLoad.set(juror.id, (jurorLoad.get(juror.id) || 0) + 1) + projectCoverage.set(project.id, (projectCoverage.get(project.id) || 0) + 1) + gapsFilled++ + } + } + + if (gapsFilled > 0) { + console.log(`[AI Assignment] Gap-filled ${gapsFilled} assignment(s) to meet coverage requirements`) + } + + return result +} + // ─── Fallback Algorithm ────────────────────────────────────────────────────── /**