feat(01-04): add auto-trigger hook + triggerAutoRank + retroactiveScan
- evaluation.ts: add triggerAutoRankIfComplete (module-level, not exported) - Checks total/completed required assignments for round - Reads autoRankOnComplete + rankingCriteria from round configJson - 5-minute cooldown guard on AUTO snapshots - Fire-and-forget via void call after isCompleted=true (never awaited) - Notifies admins via AI_RANKING_COMPLETE / AI_RANKING_FAILED - ranking.ts: add triggerAutoRank procedure (RANK-09) - Admin manual trigger reading criteria from round configJson - Creates MANUAL snapshot with QUICK mode - ranking.ts: add retroactiveScan procedure (RANK-10) - Scans ROUND_ACTIVE / ROUND_CLOSED rounds for auto-rank configured - Skips rounds with existing RETROACTIVE snapshots - Runs sequentially to avoid rate limits - ranking.ts: router now has 7 total procedures
This commit is contained in:
@@ -75,7 +75,7 @@ Phases execute in numeric order: 1 → 2 → 3 → 4 (Phase 4 can be parallelize
|
|||||||
|
|
||||||
| Phase | Plans Complete | Status | Completed |
|
| Phase | Plans Complete | Status | Completed |
|
||||||
|-------|----------------|--------|-----------|
|
|-------|----------------|--------|-----------|
|
||||||
| 1. AI Ranking Backend | 2/4 | In Progress| |
|
| 1. AI Ranking Backend | 3/4 | In Progress| |
|
||||||
| 2. Ranking Dashboard UI | 0/TBD | Not started | - |
|
| 2. Ranking Dashboard UI | 0/TBD | Not started | - |
|
||||||
| 3. Advancement + Email | 0/TBD | Not started | - |
|
| 3. Advancement + Email | 0/TBD | Not started | - |
|
||||||
| 4. Mentor Persistence | 0/TBD | Not started | - |
|
| 4. Mentor Persistence | 0/TBD | Not started | - |
|
||||||
|
|||||||
@@ -10,27 +10,27 @@ See: .planning/PROJECT.md (updated 2026-02-26)
|
|||||||
## Current Position
|
## Current Position
|
||||||
|
|
||||||
Phase: 1 of 4 (AI Ranking Backend)
|
Phase: 1 of 4 (AI Ranking Backend)
|
||||||
Plan: 2 of TBD in current phase
|
Plan: 3 of TBD in current phase
|
||||||
Status: In progress
|
Status: In progress
|
||||||
Last activity: 2026-02-26 — Plan 02 complete: AI ranking service built (ai-ranking.ts + AIAction RANKING)
|
Last activity: 2026-02-27 — Plan 03 complete: tRPC rankingRouter with 5 procedures registered in appRouter
|
||||||
|
|
||||||
Progress: [██░░░░░░░░] ~20%
|
Progress: [███░░░░░░░] ~30%
|
||||||
|
|
||||||
## Performance Metrics
|
## Performance Metrics
|
||||||
|
|
||||||
**Velocity:**
|
**Velocity:**
|
||||||
- Total plans completed: 2
|
- Total plans completed: 3
|
||||||
- Average duration: ~3 min
|
- Average duration: ~3 min
|
||||||
- Total execution time: ~6 min
|
- Total execution time: ~10 min
|
||||||
|
|
||||||
**By Phase:**
|
**By Phase:**
|
||||||
|
|
||||||
| Phase | Plans | Total | Avg/Plan |
|
| Phase | Plans | Total | Avg/Plan |
|
||||||
|-------|-------|-------|----------|
|
|-------|-------|-------|----------|
|
||||||
| 01-ai-ranking-backend | 2 | ~6 min | ~3 min |
|
| 01-ai-ranking-backend | 3 | ~10 min | ~3 min |
|
||||||
|
|
||||||
**Recent Trend:**
|
**Recent Trend:**
|
||||||
- Last 5 plans: 01-01 (~3 min), 01-02 (~3 min)
|
- Last 5 plans: 01-01 (~3 min), 01-02 (~3 min), 01-03 (~4 min)
|
||||||
- Trend: Fast (service-layer tasks)
|
- Trend: Fast (service-layer tasks)
|
||||||
|
|
||||||
*Updated after each plan completion*
|
*Updated after each plan completion*
|
||||||
@@ -51,6 +51,8 @@ Recent decisions affecting current work:
|
|||||||
- [01-02]: fetchAndRankCategory exported (not private) so tRPC router can execute pre-parsed rules without re-parsing
|
- [01-02]: fetchAndRankCategory exported (not private) so tRPC router can execute pre-parsed rules without re-parsing
|
||||||
- [01-02]: Projects with zero SUBMITTED evaluations excluded from ranking entirely (not ranked last)
|
- [01-02]: Projects with zero SUBMITTED evaluations excluded from ranking entirely (not ranked last)
|
||||||
- [01-02]: PrismaClient imported as real type (not import type) so transaction clients are compatible
|
- [01-02]: PrismaClient imported as real type (not import type) so transaction clients are compatible
|
||||||
|
- [01-03]: Typed arrays cast to Prisma.InputJsonValue via `unknown` (direct cast rejected by strict TS — no index signature)
|
||||||
|
- [01-03]: getSnapshot uses findUnique + manual TRPCError NOT_FOUND (findUniqueOrThrow gives INTERNAL_SERVER_ERROR)
|
||||||
|
|
||||||
### Pending Todos
|
### Pending Todos
|
||||||
|
|
||||||
@@ -65,5 +67,5 @@ None yet.
|
|||||||
## Session Continuity
|
## Session Continuity
|
||||||
|
|
||||||
Last session: 2026-02-27
|
Last session: 2026-02-27
|
||||||
Stopped at: Completed 01-01-PLAN.md (RankingSnapshot schema + EvaluationConfig ranking fields)
|
Stopped at: Completed 01-03-PLAN.md (tRPC rankingRouter — 5 procedures, registered in appRouter)
|
||||||
Resume file: None
|
Resume file: None
|
||||||
|
|||||||
@@ -6,6 +6,99 @@ import { notifyAdmins, NotificationTypes } from '../services/in-app-notification
|
|||||||
import { reassignAfterCOI } from './assignment'
|
import { reassignAfterCOI } from './assignment'
|
||||||
import { sendManualReminders } from '../services/evaluation-reminders'
|
import { sendManualReminders } from '../services/evaluation-reminders'
|
||||||
import { generateSummary } from '@/server/services/ai-evaluation-summary'
|
import { generateSummary } from '@/server/services/ai-evaluation-summary'
|
||||||
|
import { quickRank as aiQuickRank } from '../services/ai-ranking'
|
||||||
|
import type { EvaluationConfig } from '@/types/competition-configs'
|
||||||
|
import type { PrismaClient } from '@prisma/client'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-trigger AI ranking if all required assignments for the round are complete.
|
||||||
|
* MUST be called fire-and-forget (void). Never awaited from submission mutation.
|
||||||
|
* Implements RANK-09 cooldown guard (5-minute window).
|
||||||
|
*/
|
||||||
|
async function triggerAutoRankIfComplete(
|
||||||
|
roundId: string,
|
||||||
|
prisma: PrismaClient,
|
||||||
|
userId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
// 1. Check if round is fully evaluated
|
||||||
|
const [total, completed] = await Promise.all([
|
||||||
|
prisma.assignment.count({ where: { roundId, isRequired: true } }),
|
||||||
|
prisma.assignment.count({ where: { roundId, isRequired: true, isCompleted: true } }),
|
||||||
|
])
|
||||||
|
if (total === 0 || total !== completed) return
|
||||||
|
|
||||||
|
// 2. Check round config for auto-ranking settings
|
||||||
|
const round = await prisma.round.findUnique({
|
||||||
|
where: { id: roundId },
|
||||||
|
select: { id: true, name: true, configJson: true, competition: { select: { id: true, name: true } } },
|
||||||
|
})
|
||||||
|
if (!round) return
|
||||||
|
|
||||||
|
const config = (round.configJson as EvaluationConfig | null) ?? ({} as EvaluationConfig)
|
||||||
|
const criteriaText = config?.rankingCriteria ?? null
|
||||||
|
const autoRankEnabled = config?.autoRankOnComplete ?? false
|
||||||
|
|
||||||
|
if (!autoRankEnabled || !criteriaText) {
|
||||||
|
// Auto-ranking not configured for this round — skip silently
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Cooldown check: skip if AUTO snapshot created in last 5 minutes
|
||||||
|
const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000)
|
||||||
|
const recentSnapshot = await prisma.rankingSnapshot.findFirst({
|
||||||
|
where: { roundId, triggerType: 'AUTO', createdAt: { gte: fiveMinutesAgo } },
|
||||||
|
select: { id: true },
|
||||||
|
})
|
||||||
|
if (recentSnapshot) {
|
||||||
|
// Duplicate auto-trigger within cooldown window — skip
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Execute ranking (takes 10-30s, do not await from submission)
|
||||||
|
const result = await aiQuickRank(criteriaText, roundId, prisma, userId)
|
||||||
|
|
||||||
|
// 5. Save snapshot
|
||||||
|
await prisma.rankingSnapshot.create({
|
||||||
|
data: {
|
||||||
|
roundId,
|
||||||
|
triggeredById: null, // auto-triggered, no specific user
|
||||||
|
triggerType: 'AUTO',
|
||||||
|
criteriaText,
|
||||||
|
parsedRulesJson: result.parsedRules as unknown as import('@prisma/client').Prisma.InputJsonValue,
|
||||||
|
startupRankingJson: result.startup.rankedProjects as unknown as import('@prisma/client').Prisma.InputJsonValue,
|
||||||
|
conceptRankingJson: result.concept.rankedProjects as unknown as import('@prisma/client').Prisma.InputJsonValue,
|
||||||
|
mode: 'QUICK',
|
||||||
|
status: 'COMPLETED',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// 6. Notify admins of success
|
||||||
|
const startupCount = result.startup.rankedProjects.length
|
||||||
|
const conceptCount = result.concept.rankedProjects.length
|
||||||
|
await notifyAdmins({
|
||||||
|
type: NotificationTypes.AI_RANKING_COMPLETE,
|
||||||
|
title: 'AI Ranking Complete',
|
||||||
|
message: `Rankings generated for "${round.name}" — ${startupCount} STARTUP, ${conceptCount} BUSINESS_CONCEPT projects ranked.`,
|
||||||
|
linkUrl: `/admin/competitions/${round.competition.id}/rounds/${roundId}?tab=ranking`,
|
||||||
|
linkLabel: 'View Rankings',
|
||||||
|
priority: 'normal',
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
// Auto-trigger must never crash the calling mutation
|
||||||
|
try {
|
||||||
|
await notifyAdmins({
|
||||||
|
type: NotificationTypes.AI_RANKING_FAILED,
|
||||||
|
title: 'AI Ranking Failed',
|
||||||
|
message: `Auto-ranking failed for round (ID: ${roundId}). Please trigger manually.`,
|
||||||
|
priority: 'high',
|
||||||
|
})
|
||||||
|
} catch {
|
||||||
|
// Even notification failure must not propagate
|
||||||
|
}
|
||||||
|
console.error('[auto-rank] triggerAutoRankIfComplete failed:', error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export const evaluationRouter = router({
|
export const evaluationRouter = router({
|
||||||
/**
|
/**
|
||||||
@@ -281,6 +374,9 @@ export const evaluationRouter = router({
|
|||||||
}),
|
}),
|
||||||
])
|
])
|
||||||
|
|
||||||
|
// Auto-trigger ranking if all assignments complete (fire-and-forget, never awaited)
|
||||||
|
void triggerAutoRankIfComplete(evaluation.assignment.roundId, ctx.prisma, ctx.user.id)
|
||||||
|
|
||||||
// Audit log
|
// Audit log
|
||||||
await logAudit({
|
await logAudit({
|
||||||
prisma: ctx.prisma,
|
prisma: ctx.prisma,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import {
|
|||||||
type ParsedRankingRule,
|
type ParsedRankingRule,
|
||||||
} from '../services/ai-ranking'
|
} from '../services/ai-ranking'
|
||||||
import { logAudit } from '../utils/audit'
|
import { logAudit } from '../utils/audit'
|
||||||
|
import type { EvaluationConfig } from '@/types/competition-configs'
|
||||||
|
|
||||||
// ─── Zod Schemas ──────────────────────────────────────────────────────────────
|
// ─── Zod Schemas ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -202,4 +203,147 @@ export const rankingRouter = router({
|
|||||||
|
|
||||||
return snapshot
|
return snapshot
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* RANK-09 — Manual trigger for auto-rank (admin button on round detail page).
|
||||||
|
* Reads ranking criteria from round configJson and executes quickRank.
|
||||||
|
*/
|
||||||
|
triggerAutoRank: adminProcedure
|
||||||
|
.input(z.object({ roundId: z.string() }))
|
||||||
|
.mutation(async ({ ctx, input }) => {
|
||||||
|
const { roundId } = input
|
||||||
|
|
||||||
|
const round = await ctx.prisma.round.findUniqueOrThrow({
|
||||||
|
where: { id: roundId },
|
||||||
|
select: { id: true, name: true, configJson: true, competition: { select: { id: true } } },
|
||||||
|
})
|
||||||
|
|
||||||
|
const config = (round.configJson as EvaluationConfig | null) ?? ({} as EvaluationConfig)
|
||||||
|
const criteriaText = config?.rankingCriteria ?? null
|
||||||
|
|
||||||
|
if (!criteriaText) {
|
||||||
|
throw new TRPCError({
|
||||||
|
code: 'BAD_REQUEST',
|
||||||
|
message: 'No ranking criteria configured for this round. Add criteria in round settings first.',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await aiQuickRank(criteriaText, roundId, ctx.prisma, ctx.user.id)
|
||||||
|
|
||||||
|
const snapshot = await ctx.prisma.rankingSnapshot.create({
|
||||||
|
data: {
|
||||||
|
roundId,
|
||||||
|
triggeredById: ctx.user.id,
|
||||||
|
triggerType: 'MANUAL',
|
||||||
|
criteriaText,
|
||||||
|
parsedRulesJson: result.parsedRules as unknown as Prisma.InputJsonValue,
|
||||||
|
startupRankingJson: result.startup.rankedProjects as unknown as Prisma.InputJsonValue,
|
||||||
|
conceptRankingJson: result.concept.rankedProjects as unknown as Prisma.InputJsonValue,
|
||||||
|
mode: 'QUICK',
|
||||||
|
status: 'COMPLETED',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
await logAudit({
|
||||||
|
prisma: ctx.prisma,
|
||||||
|
userId: ctx.user.id,
|
||||||
|
action: 'RANKING_MANUAL_TRIGGERED',
|
||||||
|
entityType: 'RankingSnapshot',
|
||||||
|
entityId: snapshot.id,
|
||||||
|
detailsJson: { roundId },
|
||||||
|
ipAddress: ctx.ip,
|
||||||
|
userAgent: ctx.userAgent,
|
||||||
|
})
|
||||||
|
|
||||||
|
return { snapshotId: snapshot.id, startup: result.startup, concept: result.concept }
|
||||||
|
}),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* RANK-10 — Retroactive scan: finds all active/closed rounds with autoRankOnComplete
|
||||||
|
* configured but no RETROACTIVE snapshot yet, then executes ranking for each.
|
||||||
|
* Runs sequentially to avoid hammering OpenAI.
|
||||||
|
*/
|
||||||
|
retroactiveScan: adminProcedure
|
||||||
|
.input(z.object({}))
|
||||||
|
.mutation(async ({ ctx }) => {
|
||||||
|
const rounds = await ctx.prisma.round.findMany({
|
||||||
|
where: { status: { in: ['ROUND_ACTIVE', 'ROUND_CLOSED'] } },
|
||||||
|
select: { id: true, name: true, configJson: true },
|
||||||
|
})
|
||||||
|
|
||||||
|
const results: Array<{ roundId: string; triggered: boolean; reason?: string }> = []
|
||||||
|
|
||||||
|
for (const round of rounds) {
|
||||||
|
const config = (round.configJson as EvaluationConfig | null) ?? ({} as EvaluationConfig)
|
||||||
|
const autoRankEnabled = config?.autoRankOnComplete ?? false
|
||||||
|
const criteriaText = config?.rankingCriteria ?? null
|
||||||
|
|
||||||
|
if (!autoRankEnabled || !criteriaText) {
|
||||||
|
results.push({ roundId: round.id, triggered: false, reason: 'auto-rank not configured' })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if fully evaluated
|
||||||
|
const [total, completed] = await Promise.all([
|
||||||
|
ctx.prisma.assignment.count({ where: { roundId: round.id, isRequired: true } }),
|
||||||
|
ctx.prisma.assignment.count({ where: { roundId: round.id, isRequired: true, isCompleted: true } }),
|
||||||
|
])
|
||||||
|
|
||||||
|
if (total === 0 || total !== completed) {
|
||||||
|
results.push({
|
||||||
|
roundId: round.id,
|
||||||
|
triggered: false,
|
||||||
|
reason: `${completed}/${total} assignments complete`,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if a RETROACTIVE snapshot already exists
|
||||||
|
const existing = await ctx.prisma.rankingSnapshot.findFirst({
|
||||||
|
where: { roundId: round.id, triggerType: 'RETROACTIVE' },
|
||||||
|
select: { id: true },
|
||||||
|
})
|
||||||
|
if (existing) {
|
||||||
|
results.push({ roundId: round.id, triggered: false, reason: 'retroactive snapshot already exists' })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute ranking sequentially to avoid rate limits
|
||||||
|
try {
|
||||||
|
const result = await aiQuickRank(criteriaText, round.id, ctx.prisma, ctx.user.id)
|
||||||
|
await ctx.prisma.rankingSnapshot.create({
|
||||||
|
data: {
|
||||||
|
roundId: round.id,
|
||||||
|
triggeredById: ctx.user.id,
|
||||||
|
triggerType: 'RETROACTIVE',
|
||||||
|
criteriaText,
|
||||||
|
parsedRulesJson: result.parsedRules as unknown as Prisma.InputJsonValue,
|
||||||
|
startupRankingJson: result.startup.rankedProjects as unknown as Prisma.InputJsonValue,
|
||||||
|
conceptRankingJson: result.concept.rankedProjects as unknown as Prisma.InputJsonValue,
|
||||||
|
mode: 'QUICK',
|
||||||
|
status: 'COMPLETED',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
results.push({ roundId: round.id, triggered: true })
|
||||||
|
} catch (err) {
|
||||||
|
results.push({ roundId: round.id, triggered: false, reason: String(err) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await logAudit({
|
||||||
|
prisma: ctx.prisma,
|
||||||
|
userId: ctx.user.id,
|
||||||
|
action: 'RANKING_RETROACTIVE_SCAN',
|
||||||
|
entityType: 'Round',
|
||||||
|
detailsJson: { results },
|
||||||
|
ipAddress: ctx.ip,
|
||||||
|
userAgent: ctx.userAgent,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
results,
|
||||||
|
total: results.length,
|
||||||
|
triggered: results.filter((r) => r.triggered).length,
|
||||||
|
}
|
||||||
|
}),
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user