Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b9b6e54149 | |||
| 2a6610a019 | |||
| 19bf4530bc |
@@ -140,6 +140,10 @@ export class AgentManager extends EventEmitter {
|
||||
provider: string;
|
||||
modelId: string;
|
||||
configDir?: string;
|
||||
/** Full ordered account queue for worker-side auto-swap */
|
||||
accountQueue?: ProviderAccount[];
|
||||
/** The account ID selected for this session */
|
||||
currentAccountId?: string;
|
||||
}> {
|
||||
// Read provider accounts and priority order from settings
|
||||
const settings = readSettingsFile();
|
||||
@@ -181,6 +185,8 @@ export class AgentManager extends EventEmitter {
|
||||
provider: resolved.resolvedProvider,
|
||||
modelId: resolved.resolvedModelId,
|
||||
configDir: undefined, // Queue-based auth handles its own token refresh
|
||||
accountQueue: orderedQueue,
|
||||
currentAccountId: resolved.accountId,
|
||||
};
|
||||
}
|
||||
console.warn('[AgentManager] No available account in provider queue, falling back to legacy profile');
|
||||
@@ -413,6 +419,8 @@ export class AgentManager extends EventEmitter {
|
||||
projectId,
|
||||
processType: 'spec-creation',
|
||||
session: sessionConfig,
|
||||
accountQueue: resolved.accountQueue,
|
||||
currentAccountId: resolved.currentAccountId,
|
||||
};
|
||||
|
||||
// Store context for potential restart
|
||||
@@ -537,6 +545,8 @@ export class AgentManager extends EventEmitter {
|
||||
projectId,
|
||||
processType: 'task-execution',
|
||||
session: sessionConfig,
|
||||
accountQueue: resolved.accountQueue,
|
||||
currentAccountId: resolved.currentAccountId,
|
||||
};
|
||||
|
||||
// Store context for potential restart
|
||||
@@ -640,6 +650,8 @@ export class AgentManager extends EventEmitter {
|
||||
projectId,
|
||||
processType: 'qa-process',
|
||||
session: sessionConfig,
|
||||
accountQueue: resolved.accountQueue,
|
||||
currentAccountId: resolved.currentAccountId,
|
||||
};
|
||||
|
||||
await this.processManager.spawnWorkerProcess(taskId, executorConfig, {}, 'qa-process', projectId);
|
||||
|
||||
@@ -261,6 +261,49 @@ export class AgentProcessManager {
|
||||
allOutput: string,
|
||||
processType: ProcessType
|
||||
): boolean {
|
||||
const source = processType === 'spec-creation' ? 'roadmap' : 'task';
|
||||
|
||||
// Fast-path: check for structured outcome sentinels emitted by WorkerBridge.
|
||||
// These are prefixed with `__WORKER_OUTCOME__:<outcome>` so we can detect
|
||||
// rate limits and auth failures without relying on text pattern matching.
|
||||
if (allOutput.includes('__WORKER_OUTCOME__:rate_limited')) {
|
||||
// Build a minimal detection result that matches RateLimitDetectionResult so
|
||||
// we can reuse the existing auto-swap and notification flow.
|
||||
const syntheticRateLimitDetection = {
|
||||
isRateLimited: true,
|
||||
resetTime: undefined as string | undefined,
|
||||
limitType: undefined as 'session' | 'weekly' | undefined,
|
||||
profileId: undefined as string | undefined,
|
||||
suggestedProfile: undefined as { id: string; name: string } | undefined,
|
||||
};
|
||||
const wasHandled = this.handleRateLimitWithAutoSwap(taskId, syntheticRateLimitDetection, processType);
|
||||
if (wasHandled) return true;
|
||||
|
||||
const rateLimitInfo = createSDKRateLimitInfo(source, syntheticRateLimitDetection, { taskId });
|
||||
this.emitter.emit('sdk-rate-limit', rateLimitInfo);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (allOutput.includes('__WORKER_OUTCOME__:auth_failure')) {
|
||||
const syntheticAuthFailureDetection = {
|
||||
isAuthFailure: true,
|
||||
profileId: undefined as string | undefined,
|
||||
failureType: 'unknown' as const,
|
||||
message: 'Authentication failed (worker outcome)',
|
||||
originalError: undefined as string | undefined,
|
||||
};
|
||||
const wasHandled = this.handleAuthFailureWithAutoSwap(taskId, syntheticAuthFailureDetection);
|
||||
if (!wasHandled) {
|
||||
this.emitter.emit('auth-failure', taskId, {
|
||||
profileId: syntheticAuthFailureDetection.profileId,
|
||||
failureType: syntheticAuthFailureDetection.failureType,
|
||||
message: syntheticAuthFailureDetection.message,
|
||||
originalError: syntheticAuthFailureDetection.originalError,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
console.log('[AgentProcess] Checking for rate limit in output (last 500 chars):', allOutput.slice(-500));
|
||||
|
||||
const rateLimitDetection = detectRateLimit(allOutput);
|
||||
@@ -280,9 +323,7 @@ export class AgentProcessManager {
|
||||
);
|
||||
if (wasHandled) return true;
|
||||
|
||||
const source = processType === 'spec-creation' ? 'roadmap' : 'task';
|
||||
const rateLimitInfo = createSDKRateLimitInfo(source, rateLimitDetection, { taskId });
|
||||
console.log('[AgentProcess] Emitting sdk-rate-limit event (manual):', rateLimitInfo);
|
||||
this.emitter.emit('sdk-rate-limit', rateLimitInfo);
|
||||
return true;
|
||||
}
|
||||
@@ -872,7 +913,14 @@ export class AgentProcessManager {
|
||||
}
|
||||
});
|
||||
|
||||
let workerErrorOutput = '';
|
||||
const MAX_ERROR_OUTPUT_LENGTH = 10_000;
|
||||
|
||||
bridge.on('error', (tId: string, error: string, pId?: string) => {
|
||||
workerErrorOutput += error + '\n';
|
||||
if (workerErrorOutput.length > MAX_ERROR_OUTPUT_LENGTH) {
|
||||
workerErrorOutput = workerErrorOutput.slice(-MAX_ERROR_OUTPUT_LENGTH);
|
||||
}
|
||||
this.emitter.emit('error', tId, error, pId);
|
||||
});
|
||||
|
||||
@@ -893,16 +941,17 @@ export class AgentProcessManager {
|
||||
}
|
||||
|
||||
if (code !== 0) {
|
||||
// Collect any output for rate limit / auth failure detection
|
||||
// For worker threads, error messages are emitted via 'error' events
|
||||
// rather than stdout parsing. The handleProcessFailure method still works
|
||||
// with accumulated output if needed.
|
||||
this.emitter.emit('execution-progress', tId, {
|
||||
phase: 'failed',
|
||||
phaseProgress: 0,
|
||||
overallProgress: 0,
|
||||
message: `Worker exited with code ${code}`,
|
||||
}, pId);
|
||||
// Attempt rate limit / auth failure detection and auto-swap
|
||||
const wasHandled = this.handleProcessFailure(tId, workerErrorOutput, processType);
|
||||
|
||||
if (!wasHandled) {
|
||||
this.emitter.emit('execution-progress', tId, {
|
||||
phase: 'failed',
|
||||
phaseProgress: 0,
|
||||
overallProgress: 0,
|
||||
message: `Worker exited with code ${code}`,
|
||||
}, pId);
|
||||
}
|
||||
}
|
||||
|
||||
this.emitter.emit('exit', tId, code, pType, pId);
|
||||
@@ -931,7 +980,7 @@ export class AgentProcessManager {
|
||||
|
||||
// Emit initial progress
|
||||
this.emitter.emit('execution-progress', taskId, {
|
||||
phase: processType === 'spec-creation' ? 'planning' : 'planning',
|
||||
phase: 'planning',
|
||||
phaseProgress: 0,
|
||||
overallProgress: 0,
|
||||
message: 'Starting AI agent session...',
|
||||
|
||||
@@ -264,7 +264,7 @@ export class AgentQueueManager {
|
||||
abortSignal: abortController.signal,
|
||||
},
|
||||
(event: IdeationStreamEvent) => {
|
||||
if (event.type === 'text-delta') {
|
||||
if (event.type === 'text-delta' || event.type === 'status') {
|
||||
this.emitter.emit('ideation-log', projectId, event.text);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
import type { ExecutionProgressData, ProcessType } from '../../../main/agent/types';
|
||||
import type { SessionConfig, SessionResult, StreamEvent } from '../session/types';
|
||||
import type { RunnerOptions } from '../session/runner';
|
||||
import type { ProviderAccount } from '../../../shared/types/provider-account';
|
||||
|
||||
// =============================================================================
|
||||
// Worker Configuration
|
||||
@@ -28,6 +29,10 @@ export interface WorkerConfig {
|
||||
processType: ProcessType;
|
||||
/** Serializable session config (model resolved in worker from these params) */
|
||||
session: SerializableSessionConfig;
|
||||
/** Account queue for auto-swap on rate limit (resolved in main thread, serializable) */
|
||||
accountQueue?: ProviderAccount[];
|
||||
/** Current account ID for the session (used to exclude on swap) */
|
||||
currentAccountId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -189,4 +194,8 @@ export interface AgentExecutorConfig {
|
||||
session: SerializableSessionConfig;
|
||||
/** Optional auth refresh callback (runs in main thread) */
|
||||
onAuthRefresh?: RunnerOptions['onAuthRefresh'];
|
||||
/** Account queue for auto-swap (resolved in main thread where settings are available) */
|
||||
accountQueue?: ProviderAccount[];
|
||||
/** Current account ID from queue resolution */
|
||||
currentAccountId?: string;
|
||||
}
|
||||
|
||||
@@ -92,6 +92,8 @@ export class WorkerBridge extends EventEmitter {
|
||||
projectId: config.projectId,
|
||||
processType: config.processType,
|
||||
session: config.session,
|
||||
accountQueue: config.accountQueue,
|
||||
currentAccountId: config.currentAccountId,
|
||||
};
|
||||
|
||||
const workerPath = resolveWorkerPath();
|
||||
@@ -212,6 +214,11 @@ export class WorkerBridge extends EventEmitter {
|
||||
/**
|
||||
* Handle the final session result from the worker.
|
||||
* Maps SessionResult.outcome to an exit code.
|
||||
*
|
||||
* For `rate_limited` and `auth_failure` outcomes, the error message is
|
||||
* prefixed with a structured sentinel (`__WORKER_OUTCOME__:<outcome>`) so
|
||||
* that AgentProcess can detect the failure type structurally instead of
|
||||
* relying solely on text pattern matching.
|
||||
*/
|
||||
private handleResult(taskId: string, result: SessionResult, projectId?: string): void {
|
||||
// Map outcome to exit code
|
||||
@@ -222,7 +229,18 @@ export class WorkerBridge extends EventEmitter {
|
||||
this.emitTyped('log', taskId, summary, projectId);
|
||||
|
||||
if (result.error) {
|
||||
this.emitTyped('error', taskId, result.error.message, projectId);
|
||||
// For structured outcomes that AgentProcess needs to react to, prefix the
|
||||
// error message with a sentinel so the consuming layer can detect the
|
||||
// outcome type without pattern-matching on human-readable text.
|
||||
const needsSentinel = result.outcome === 'rate_limited' || result.outcome === 'auth_failure';
|
||||
const errorMessage = needsSentinel
|
||||
? `__WORKER_OUTCOME__:${result.outcome} ${result.error.message}`
|
||||
: result.error.message;
|
||||
this.emitTyped('error', taskId, errorMessage, projectId);
|
||||
} else if (result.outcome === 'rate_limited' || result.outcome === 'auth_failure') {
|
||||
// No error object but the outcome still indicates a structured failure —
|
||||
// emit the sentinel alone so AgentProcess can trigger the swap flow.
|
||||
this.emitTyped('error', taskId, `__WORKER_OUTCOME__:${result.outcome}`, projectId);
|
||||
}
|
||||
|
||||
// Emit exit and cleanup
|
||||
|
||||
@@ -34,7 +34,9 @@ import type {
|
||||
WorkerTaskEventMessage,
|
||||
} from './types';
|
||||
import type { Tool as AITool } from 'ai';
|
||||
import type { SessionConfig, StreamEvent, SessionResult } from '../session/types';
|
||||
import type { SessionConfig, StreamEvent, SessionResult, SessionError } from '../session/types';
|
||||
import type { QueueResolvedAuth } from '../auth/types';
|
||||
import type { ProviderAccount } from '../../../shared/types/provider-account';
|
||||
import { BuildOrchestrator } from '../orchestration/build-orchestrator';
|
||||
import { QALoop } from '../orchestration/qa-loop';
|
||||
import { SpecOrchestrator } from '../orchestration/spec-orchestrator';
|
||||
@@ -122,6 +124,24 @@ parentPort.on('message', (msg: MainToWorkerMessage) => {
|
||||
// Shared Helpers
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Build the onAccountSwitch callback for runner options.
|
||||
* Extracted to avoid triplication across the three session-launch call sites.
|
||||
* The second parameter (error) is accepted for type correctness but not used yet.
|
||||
*/
|
||||
function buildAccountSwitchCallback(
|
||||
modelId: string,
|
||||
accountQueue?: ProviderAccount[],
|
||||
): ((failedAccountId: string, error: SessionError) => Promise<QueueResolvedAuth | null>) | undefined {
|
||||
if (!accountQueue?.length) return undefined;
|
||||
return async (failedAccountId: string) => {
|
||||
const { resolveAuthFromQueue } = await import('../auth/resolver');
|
||||
return resolveAuthFromQueue(modelId, accountQueue, {
|
||||
excludeAccountIds: [failedAccountId],
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconstruct the SecurityProfile from the serialized form in session config.
|
||||
* SecurityProfile uses Set objects that can't cross worker boundaries.
|
||||
@@ -341,6 +361,8 @@ async function runSingleSession(
|
||||
modelId: phaseModelId,
|
||||
})
|
||||
: undefined,
|
||||
onAccountSwitch: buildAccountSwitchCallback(baseSession.modelId, config.accountQueue),
|
||||
currentAccountId: config.currentAccountId,
|
||||
};
|
||||
|
||||
let sessionResult: SessionResult;
|
||||
@@ -517,6 +539,8 @@ async function runDefaultSession(
|
||||
modelId: session.modelId,
|
||||
})
|
||||
: undefined,
|
||||
onAccountSwitch: buildAccountSwitchCallback(session.modelId, config.accountQueue),
|
||||
currentAccountId: config.currentAccountId,
|
||||
}, {
|
||||
contextWindowLimit,
|
||||
apiKey: session.apiKey,
|
||||
@@ -1113,6 +1137,8 @@ async function runAgenticSpecOrchestrator(
|
||||
modelId: session.modelId,
|
||||
})
|
||||
: undefined,
|
||||
onAccountSwitch: buildAccountSwitchCallback(session.modelId, config.accountQueue),
|
||||
currentAccountId: config.currentAccountId,
|
||||
}, {
|
||||
contextWindowLimit,
|
||||
apiKey: session.apiKey,
|
||||
|
||||
@@ -28,6 +28,8 @@ import type { McpClientResult } from '../mcp/types';
|
||||
import { createProvider, detectProviderFromModel } from '../providers/factory';
|
||||
import { buildToolRegistry } from '../tools/build-registry';
|
||||
import type { QueueResolvedAuth } from '../auth/types';
|
||||
import { wrapWithAutoSwap } from '../providers/auto-swap-middleware';
|
||||
import { notifyRendererOfSwap } from './swap-notifier';
|
||||
import type {
|
||||
AgentClientConfig,
|
||||
AgentClientResult,
|
||||
@@ -180,8 +182,14 @@ export async function createAgentClient(
|
||||
await closeAllMcpClients(mcpClients);
|
||||
};
|
||||
|
||||
// Wrap model with auto-swap middleware for transparent rate-limit failover
|
||||
console.log('[AutoSwap] createAgentClient for', agentType, '| account:', queueAuth?.accountId ?? 'legacy', '| queue:', queueConfig ? queueConfig.queue.length + ' accounts' : 'none');
|
||||
const wrappedModel = queueConfig
|
||||
? wrapWithAutoSwap(model, queueAuth, queueConfig.queue, queueConfig.requestedModel, notifyRendererOfSwap)
|
||||
: model;
|
||||
|
||||
return {
|
||||
model,
|
||||
model: wrappedModel,
|
||||
tools,
|
||||
mcpClients,
|
||||
systemPrompt,
|
||||
@@ -285,8 +293,14 @@ export async function createSimpleClient(
|
||||
});
|
||||
}
|
||||
|
||||
// Wrap model with auto-swap middleware for transparent rate-limit failover
|
||||
console.log('[AutoSwap] createSimpleClient | account:', queueAuth?.accountId ?? 'legacy', '| model:', resolvedModelId, '| queue:', queueConfig ? queueConfig.queue.length + ' accounts' : 'none');
|
||||
const wrappedModel = queueConfig
|
||||
? wrapWithAutoSwap(model, queueAuth, queueConfig.queue, queueConfig.requestedModel, notifyRendererOfSwap)
|
||||
: model;
|
||||
|
||||
return {
|
||||
model,
|
||||
model: wrappedModel,
|
||||
resolvedModelId,
|
||||
tools,
|
||||
systemPrompt,
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* Swap Notifier
|
||||
* =============
|
||||
*
|
||||
* Sends a UI notification when the auto-swap middleware switches accounts.
|
||||
* Bridges the AI middleware layer (no IPC access) to the renderer via
|
||||
* Electron's BrowserWindow.
|
||||
*/
|
||||
|
||||
import { BrowserWindow } from 'electron';
|
||||
import { IPC_CHANNELS } from '../../../shared/constants/ipc';
|
||||
import type { AutoSwapEvent } from '../providers/auto-swap-middleware';
|
||||
|
||||
/**
|
||||
* Notify the renderer that the middleware swapped to a different account.
|
||||
* Uses the existing PROACTIVE_SWAP_NOTIFICATION channel so the
|
||||
* ProactiveSwapListener component shows the swap toast.
|
||||
*/
|
||||
export function notifyRendererOfSwap(event: AutoSwapEvent): void {
|
||||
console.log('[AutoSwap] Sending UI notification:', event.fromAccountName, '→', event.toAccountName);
|
||||
const mainWindow = BrowserWindow.getAllWindows()[0];
|
||||
if (!mainWindow) {
|
||||
console.log('[AutoSwap] No BrowserWindow found — UI notification skipped');
|
||||
return;
|
||||
}
|
||||
|
||||
mainWindow.webContents.send(IPC_CHANNELS.PROACTIVE_SWAP_NOTIFICATION, {
|
||||
fromProfile: { id: event.fromAccountId, name: event.fromAccountName },
|
||||
toProfile: { id: event.toAccountId, name: event.toAccountName },
|
||||
reason: 'reactive-middleware',
|
||||
});
|
||||
console.log('[AutoSwap] UI notification sent via PROACTIVE_SWAP_NOTIFICATION channel');
|
||||
}
|
||||
@@ -99,25 +99,6 @@ const PUPPETEER_SERVER: McpServerConfig = {
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Auto-Claude MCP server - custom build management tools.
|
||||
* Used by planner, coder, and QA agents for build progress tracking.
|
||||
*/
|
||||
function createAutoClaudeServer(specDir: string): McpServerConfig {
|
||||
return {
|
||||
id: 'auto-claude',
|
||||
name: 'Auto-Claude',
|
||||
description: 'Build management tools (progress tracking, session context)',
|
||||
enabledByDefault: true,
|
||||
transport: {
|
||||
type: 'stdio',
|
||||
command: 'node',
|
||||
args: ['auto-claude-mcp-server.js'],
|
||||
env: { SPEC_DIR: specDir },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Registry
|
||||
// =============================================================================
|
||||
@@ -176,8 +157,11 @@ export function getMcpServerConfig(
|
||||
return PUPPETEER_SERVER;
|
||||
|
||||
case 'auto-claude': {
|
||||
const specDir = options.specDir ?? '';
|
||||
return createAutoClaudeServer(specDir);
|
||||
// Tools are implemented as builtin TypeScript tools in tools/auto-claude/ and
|
||||
// injected directly into agent sessions. The external Node.js subprocess is
|
||||
// obsolete since the migration from Python to the Vercel AI SDK.
|
||||
// resolveMcpServers() filters out null results automatically.
|
||||
return null;
|
||||
}
|
||||
|
||||
default:
|
||||
|
||||
@@ -272,16 +272,36 @@ export class BuildOrchestrator extends EventEmitter {
|
||||
`Implementation plan is invalid and cannot be executed: ${errorDetail}`);
|
||||
}
|
||||
|
||||
// Check if build is already complete
|
||||
if (await this.isBuildComplete()) {
|
||||
this.transitionPhase('complete', 'Build already complete');
|
||||
return this.buildOutcome(true, Date.now() - startTime);
|
||||
// Guard: if all subtasks appear "completed" but no coding has actually happened,
|
||||
// the spec pipeline or planner marked them erroneously. Reset to "pending".
|
||||
// This catches the case where the spec orchestrator created the plan (isFirstRun=false)
|
||||
// but the planner LLM set all subtasks to "completed" during planning.
|
||||
let buildComplete = await this.isBuildComplete();
|
||||
if (buildComplete) {
|
||||
const hasCodedBefore = await this.hasCodingEvidence();
|
||||
if (!hasCodedBefore) {
|
||||
this.emitTyped('log', 'All subtasks marked completed but no coding evidence found — resetting to pending');
|
||||
await this.resetSubtaskStatuses();
|
||||
buildComplete = false; // We just reset, so it's no longer complete
|
||||
} else {
|
||||
// Coding happened — check if QA already passed before declaring complete.
|
||||
// This handles the restart-after-QA-failure case: skip coding, re-run QA.
|
||||
const qaStatus = await this.readQAStatus();
|
||||
if (qaStatus === 'passed') {
|
||||
this.transitionPhase('complete', 'Build already complete');
|
||||
return this.buildOutcome(true, Date.now() - startTime);
|
||||
}
|
||||
// QA failed or never ran — skip coding, fall through to QA
|
||||
this.emitTyped('log', `Coding complete but QA status is "${qaStatus}" — skipping to QA`);
|
||||
}
|
||||
}
|
||||
|
||||
// Coding phase
|
||||
const codingResult = await this.runCodingPhase();
|
||||
if (!codingResult.success) {
|
||||
return this.buildOutcome(false, Date.now() - startTime, codingResult.error);
|
||||
// Coding phase (skip if all subtasks already completed with coding evidence)
|
||||
if (!buildComplete) {
|
||||
const codingResult = await this.runCodingPhase();
|
||||
if (!codingResult.success) {
|
||||
return this.buildOutcome(false, Date.now() - startTime, codingResult.error);
|
||||
}
|
||||
}
|
||||
|
||||
// QA review phase
|
||||
@@ -526,6 +546,10 @@ export class BuildOrchestrator extends EventEmitter {
|
||||
return { success: false, error: 'Build cancelled' };
|
||||
}
|
||||
|
||||
if (reviewResult.outcome === 'error' || reviewResult.outcome === 'auth_failure' || reviewResult.outcome === 'rate_limited') {
|
||||
return { success: false, error: reviewResult.error?.message ?? 'QA review session failed' };
|
||||
}
|
||||
|
||||
// Check QA result
|
||||
const qaStatus = await this.readQAStatus();
|
||||
|
||||
@@ -562,6 +586,15 @@ export class BuildOrchestrator extends EventEmitter {
|
||||
});
|
||||
|
||||
this.emitTyped('session-complete', fixResult, 'qa_fixing');
|
||||
|
||||
if (fixResult.outcome === 'cancelled') {
|
||||
return { success: false, error: 'Build cancelled' };
|
||||
}
|
||||
|
||||
if (fixResult.outcome === 'error' || fixResult.outcome === 'auth_failure' || fixResult.outcome === 'rate_limited') {
|
||||
return { success: false, error: fixResult.error?.message ?? 'QA fix session failed' };
|
||||
}
|
||||
|
||||
this.markPhaseCompleted('qa_fixing');
|
||||
|
||||
// Delete qa_report.md before re-review so the reviewer writes a clean verdict.
|
||||
@@ -684,6 +717,26 @@ export class BuildOrchestrator extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any coding work has actually been performed.
|
||||
* Uses task_logs.json as evidence — if the coding phase has entries,
|
||||
* real work was done. If not, the subtask statuses are from the planner.
|
||||
*/
|
||||
private async hasCodingEvidence(): Promise<boolean> {
|
||||
const taskLogsPath = join(this.config.specDir, 'task_logs.json');
|
||||
try {
|
||||
const raw = await readFile(taskLogsPath, 'utf-8');
|
||||
const logs = JSON.parse(raw);
|
||||
// Check if coding phase has any entries
|
||||
if (logs?.phases?.coding?.entries?.length > 0) return true;
|
||||
if (logs?.phases?.coding?.status === 'completed') return true;
|
||||
return false;
|
||||
} catch {
|
||||
// No task_logs.json or invalid — no coding evidence
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if all subtasks in the implementation plan are completed.
|
||||
*/
|
||||
|
||||
@@ -255,6 +255,10 @@ export class QALoop extends EventEmitter {
|
||||
return this.outcome(false, iteration, Date.now() - startTime, 'cancelled');
|
||||
}
|
||||
|
||||
if (reviewResult.outcome === 'rate_limited') {
|
||||
return this.outcome(false, iteration, Date.now() - startTime, 'error', reviewResult.error?.message ?? 'Rate limited during QA review');
|
||||
}
|
||||
|
||||
// Read QA signoff from implementation_plan.json
|
||||
const signoff = await this.readQASignoff();
|
||||
const status = this.resolveQAStatus(signoff);
|
||||
@@ -318,7 +322,7 @@ export class QALoop extends EventEmitter {
|
||||
return this.outcome(false, iteration, Date.now() - startTime, 'cancelled');
|
||||
}
|
||||
|
||||
if (fixResult.outcome === 'error' || fixResult.outcome === 'auth_failure') {
|
||||
if (fixResult.outcome === 'error' || fixResult.outcome === 'auth_failure' || fixResult.outcome === 'rate_limited') {
|
||||
this.emitTyped('log', `Fixer error: ${fixResult.error?.message ?? 'unknown'}`);
|
||||
await this.writeReports('max_iterations');
|
||||
return this.outcome(false, iteration, Date.now() - startTime, 'error', fixResult.error?.message);
|
||||
|
||||
@@ -553,7 +553,7 @@ export class SpecOrchestrator extends EventEmitter {
|
||||
errors.push(errorMsg);
|
||||
|
||||
// Non-retryable errors
|
||||
if (result.outcome === 'auth_failure') {
|
||||
if (result.outcome === 'auth_failure' || result.outcome === 'rate_limited') {
|
||||
return { phase, success: false, errors, retries: attempt };
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* Auto-Swap Middleware
|
||||
* ====================
|
||||
*
|
||||
* AI SDK LanguageModel middleware that intercepts rate limit errors (429)
|
||||
* and automatically retries with the next available account from the
|
||||
* global priority queue.
|
||||
*
|
||||
* Applied via wrapLanguageModel() in createSimpleClient() and createAgentClient(),
|
||||
* giving auto-swap to ALL runners without per-runner changes.
|
||||
*/
|
||||
|
||||
import { wrapLanguageModel } from 'ai';
|
||||
import type { LanguageModelMiddleware, LanguageModel } from 'ai';
|
||||
import type { LanguageModelV3 } from '@ai-sdk/provider';
|
||||
import { resolveAuthFromQueue } from '../auth/resolver';
|
||||
import { createProvider } from './factory';
|
||||
import type { QueueResolvedAuth } from '../auth/types';
|
||||
import type { ProviderAccount } from '../../../shared/types/provider-account';
|
||||
import { isRateLimitError } from '../session/error-classifier';
|
||||
import { extractWaitDuration, formatWaitDuration, sleepWithAbort } from '../session/rate-limit-wait';
|
||||
|
||||
/** Details emitted when the middleware swaps to a different account. */
|
||||
export interface AutoSwapEvent {
|
||||
fromAccountId: string;
|
||||
fromAccountName: string;
|
||||
toAccountId: string;
|
||||
toAccountName: string;
|
||||
}
|
||||
|
||||
interface AutoSwapContext {
|
||||
queueAuth: QueueResolvedAuth;
|
||||
queue: ProviderAccount[];
|
||||
requestedModel: string;
|
||||
/** Called after a successful swap so the UI can reflect the change. */
|
||||
onSwap?: (event: AutoSwapEvent) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared rate-limit recovery logic for wrapGenerate and wrapStream.
|
||||
*
|
||||
* On a 429 error:
|
||||
* 1. Try swapping to another account in the queue.
|
||||
* 2. If no swap target, wait for the session rate limit to reset (null = weekly limit, rethrow).
|
||||
* 3. Retry with `retry` (original model) or `retryOnNew` (new model, needs params).
|
||||
*/
|
||||
async function handleRateLimit<T>(
|
||||
error: unknown,
|
||||
ctx: AutoSwapContext,
|
||||
swappedRef: { value: boolean },
|
||||
retry: () => PromiseLike<T>,
|
||||
retryOnNew: (newModel: LanguageModelV3) => PromiseLike<T>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<T> {
|
||||
if (swappedRef.value) {
|
||||
console.log('[AutoSwap] Already swapped once this call — not retrying again (prevents A→B→A loop)');
|
||||
throw error;
|
||||
}
|
||||
if (!isRateLimitError(error)) {
|
||||
console.log('[AutoSwap] Error is not a rate limit — passing through:', error instanceof Error ? error.message : String(error));
|
||||
throw error;
|
||||
}
|
||||
|
||||
swappedRef.value = true;
|
||||
console.log('[AutoSwap] 🔴 Rate limit hit on account:', ctx.queueAuth.accountId, '| model:', ctx.requestedModel);
|
||||
console.log('[AutoSwap] Queue has', ctx.queue.length, 'accounts. Excluding current, looking for swap target...');
|
||||
|
||||
const newAuth = await resolveAuthFromQueue(ctx.requestedModel, ctx.queue, {
|
||||
excludeAccountIds: [ctx.queueAuth.accountId],
|
||||
});
|
||||
|
||||
if (!newAuth) {
|
||||
// No swap target — wait for session rate limit reset
|
||||
console.log('[AutoSwap] No swap target available — all other accounts excluded or unavailable');
|
||||
const waitInfo = extractWaitDuration(error);
|
||||
if (!waitInfo) {
|
||||
console.log('[AutoSwap] Cannot extract wait duration (weekly limit?) — giving up');
|
||||
throw error;
|
||||
}
|
||||
console.log('[AutoSwap] Waiting', formatWaitDuration(waitInfo.waitMs), 'for rate limit reset (type:', waitInfo.limitType + ')');
|
||||
await sleepWithAbort(waitInfo.waitMs, signal);
|
||||
console.log('[AutoSwap] Wait complete — retrying with original account');
|
||||
return retry();
|
||||
}
|
||||
|
||||
// Notify the UI about the swap
|
||||
const fromAccount = ctx.queue.find((a) => a.id === ctx.queueAuth.accountId);
|
||||
const toAccount = ctx.queue.find((a) => a.id === newAuth.accountId);
|
||||
const fromName = fromAccount?.name ?? ctx.queueAuth.accountId;
|
||||
const toName = toAccount?.name ?? newAuth.accountId;
|
||||
console.log('[AutoSwap] ✅ SWAPPING:', fromName, '→', toName, '| provider:', newAuth.resolvedProvider, '| model:', newAuth.resolvedModelId);
|
||||
|
||||
ctx.onSwap?.({
|
||||
fromAccountId: ctx.queueAuth.accountId,
|
||||
fromAccountName: fromName,
|
||||
toAccountId: newAuth.accountId,
|
||||
toAccountName: toName,
|
||||
});
|
||||
|
||||
const newModel = createProvider({
|
||||
config: {
|
||||
provider: newAuth.resolvedProvider,
|
||||
apiKey: newAuth.apiKey,
|
||||
baseURL: newAuth.baseURL,
|
||||
headers: newAuth.headers,
|
||||
oauthTokenFilePath: newAuth.oauthTokenFilePath,
|
||||
},
|
||||
modelId: newAuth.resolvedModelId,
|
||||
}) as LanguageModelV3;
|
||||
|
||||
console.log('[AutoSwap] New model created — retrying request on', toName);
|
||||
return retryOnNew(newModel);
|
||||
}
|
||||
|
||||
function createAutoSwapMiddleware(ctx: AutoSwapContext): LanguageModelMiddleware {
|
||||
const swapped = { value: false }; // Only swap once per model instance
|
||||
|
||||
return {
|
||||
specificationVersion: 'v3' as const,
|
||||
wrapGenerate: async ({ doGenerate, params }) => {
|
||||
// Reset per-call so each generateText() gets its own swap opportunity.
|
||||
// The flag only prevents infinite A→B→A loops within a single call.
|
||||
swapped.value = false;
|
||||
try {
|
||||
return await doGenerate();
|
||||
} catch (error) {
|
||||
console.log('[AutoSwap] wrapGenerate caught error — entering handleRateLimit');
|
||||
return handleRateLimit(error, ctx, swapped, doGenerate, (m) => m.doGenerate(params), params.abortSignal);
|
||||
}
|
||||
},
|
||||
wrapStream: async ({ doStream, params }) => {
|
||||
swapped.value = false;
|
||||
try {
|
||||
return await doStream();
|
||||
} catch (error) {
|
||||
console.log('[AutoSwap] wrapStream caught error — entering handleRateLimit');
|
||||
return handleRateLimit(error, ctx, swapped, doStream, (m) => m.doStream(params), params.abortSignal);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a model with auto-swap middleware if queue context is available.
|
||||
* When a 429 error occurs, the middleware resolves the next available
|
||||
* account and retries with a new model — transparent to the caller.
|
||||
*
|
||||
* No-op if queue has fewer than 2 accounts (nothing to swap to).
|
||||
*/
|
||||
export function wrapWithAutoSwap(
|
||||
model: LanguageModel,
|
||||
queueAuth: QueueResolvedAuth | null,
|
||||
queue: ProviderAccount[],
|
||||
requestedModel: string,
|
||||
onSwap?: (event: AutoSwapEvent) => void,
|
||||
): LanguageModel {
|
||||
if (!queueAuth || queue.length < 2) {
|
||||
console.log('[AutoSwap] Middleware NOT applied — queue has', queue.length, 'account(s) (need ≥2 for swap)');
|
||||
return model;
|
||||
}
|
||||
console.log('[AutoSwap] Middleware applied — account:', queueAuth.accountId, '| model:', requestedModel, '| queue size:', queue.length);
|
||||
return wrapLanguageModel({
|
||||
model: model as LanguageModelV3,
|
||||
middleware: createAutoSwapMiddleware({ queueAuth, queue, requestedModel, onSwap }),
|
||||
});
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { generateText } from 'ai';
|
||||
|
||||
import { createSimpleClient } from '../client/factory';
|
||||
import type { ModelShorthand, ThinkingLevel } from '../config/types';
|
||||
import { withRateLimitRetry } from '../session/rate-limit-retry';
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
@@ -137,11 +138,11 @@ export async function generateChangelog(
|
||||
thinkingLevel,
|
||||
});
|
||||
|
||||
const result = await generateText({
|
||||
const result = await withRateLimitRetry(() => generateText({
|
||||
model: client.model,
|
||||
system: client.systemPrompt,
|
||||
prompt,
|
||||
});
|
||||
}));
|
||||
|
||||
if (result.text.trim()) {
|
||||
return { success: true, text: result.text.trim() };
|
||||
|
||||
@@ -23,6 +23,8 @@ import type { Tool as AITool } from 'ai';
|
||||
import * as crypto from 'node:crypto';
|
||||
|
||||
import { createSimpleClient } from '../../client/factory';
|
||||
import { withRateLimitRetry } from '../../session/rate-limit-retry';
|
||||
import { formatWaitDuration } from '../../session/rate-limit-wait';
|
||||
import type { SimpleClientResult } from '../../client/types';
|
||||
import type { ModelShorthand, ThinkingLevel } from '../../config/types';
|
||||
import { buildThinkingProviderOptions } from '../../config/types';
|
||||
@@ -557,43 +559,58 @@ export class ParallelOrchestratorReviewer {
|
||||
let stepCount = 0;
|
||||
let toolCallCount = 0;
|
||||
const toolsUsed = new Set<string>();
|
||||
const { structuredOutput, textOutput } = await withRateLimitRetry(async () => {
|
||||
// Reset counters on each retry attempt
|
||||
stepCount = 0;
|
||||
toolCallCount = 0;
|
||||
toolsUsed.clear();
|
||||
|
||||
// Use streamText instead of generateText — Codex endpoint only supports streaming.
|
||||
// Output.object() generates structured output as a final step after all tool calls.
|
||||
const stream = streamText({
|
||||
model: client.model,
|
||||
system: genOptions.system,
|
||||
messages: [{ role: 'user' as const, content: userMessage }],
|
||||
tools,
|
||||
stopWhen: stepCountIs(100),
|
||||
output: Output.object({ schema: SpecialistOutputOutputSchema }),
|
||||
abortSignal,
|
||||
...(genOptions.providerOptions ? { providerOptions: genOptions.providerOptions } : {}),
|
||||
onStepFinish: ({ toolCalls }) => {
|
||||
stepCount++;
|
||||
if (toolCalls && toolCalls.length > 0) {
|
||||
for (const tc of toolCalls) {
|
||||
toolCallCount++;
|
||||
toolsUsed.add(tc.toolName);
|
||||
// Use streamText instead of generateText — Codex endpoint only supports streaming.
|
||||
// Output.object() generates structured output as a final step after all tool calls.
|
||||
const stream = streamText({
|
||||
model: client.model,
|
||||
system: genOptions.system,
|
||||
messages: [{ role: 'user' as const, content: userMessage }],
|
||||
tools,
|
||||
stopWhen: stepCountIs(100),
|
||||
output: Output.object({ schema: SpecialistOutputOutputSchema }),
|
||||
abortSignal,
|
||||
...(genOptions.providerOptions ? { providerOptions: genOptions.providerOptions } : {}),
|
||||
onStepFinish: ({ toolCalls }) => {
|
||||
stepCount++;
|
||||
if (toolCalls && toolCalls.length > 0) {
|
||||
for (const tc of toolCalls) {
|
||||
toolCallCount++;
|
||||
toolsUsed.add(tc.toolName);
|
||||
}
|
||||
this.reportProgress({
|
||||
phase: config.name,
|
||||
progress: 40,
|
||||
message: `[Specialist:${config.name}] Step ${stepCount}: ${toolCalls.length} tool call(s) — ${toolCalls.map((tc) => tc.toolName).join(', ')}`,
|
||||
prNumber: context.prNumber,
|
||||
});
|
||||
}
|
||||
this.reportProgress({
|
||||
phase: config.name,
|
||||
progress: 40,
|
||||
message: `[Specialist:${config.name}] Step ${stepCount}: ${toolCalls.length} tool call(s) — ${toolCalls.map((tc) => tc.toolName).join(', ')}`,
|
||||
prNumber: context.prNumber,
|
||||
});
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Consume the stream (required before accessing output/text)
|
||||
for await (const _part of stream.fullStream) { /* consume */ }
|
||||
|
||||
return { structuredOutput: await stream.output, textOutput: await stream.text };
|
||||
}, {
|
||||
signal: abortSignal,
|
||||
onWaiting: (waitMs) => this.reportProgress({
|
||||
phase: config.name,
|
||||
progress: 38,
|
||||
message: `[Specialist:${config.name}] Rate limited. Waiting ${formatWaitDuration(waitMs)}...`,
|
||||
prNumber: context.prNumber,
|
||||
}),
|
||||
});
|
||||
|
||||
// Consume the stream (required before accessing output/text)
|
||||
for await (const _part of stream.fullStream) { /* consume */ }
|
||||
|
||||
// Use structured output if available, fall back to text parsing
|
||||
const structuredOutput = await stream.output;
|
||||
const findings = structuredOutput
|
||||
? parseSpecialistOutput(config.name, structuredOutput)
|
||||
: parseSpecialistOutput(config.name, await stream.text);
|
||||
: parseSpecialistOutput(config.name, textOutput);
|
||||
|
||||
const toolSummary = toolCallCount > 0
|
||||
? ` (${toolCallCount} tool calls: ${Array.from(toolsUsed).join(', ')})`
|
||||
@@ -755,41 +772,53 @@ Validate each finding by reading the actual code at the specified file and line.
|
||||
try {
|
||||
let validatorToolCalls = 0;
|
||||
|
||||
// Use streamText — Codex endpoint only supports streaming.
|
||||
// Output.object() generates the validation array (wrapped in { validations: [...] }) as a final step.
|
||||
const stream = streamText({
|
||||
model: client.model,
|
||||
system: genOptions.system,
|
||||
messages: [{ role: 'user' as const, content: userMessage }],
|
||||
tools,
|
||||
stopWhen: stepCountIs(150),
|
||||
output: Output.object({ schema: FindingValidationsOutputSchema }),
|
||||
abortSignal,
|
||||
...(genOptions.providerOptions ? { providerOptions: genOptions.providerOptions } : {}),
|
||||
onStepFinish: ({ toolCalls }) => {
|
||||
if (toolCalls && toolCalls.length > 0) {
|
||||
validatorToolCalls += toolCalls.length;
|
||||
this.reportProgress({
|
||||
phase: 'validation',
|
||||
progress: 75,
|
||||
message: `[FindingValidator] Examining code: ${toolCalls.map((tc) => tc.toolName).join(', ')}`,
|
||||
prNumber: context.prNumber,
|
||||
});
|
||||
}
|
||||
},
|
||||
const { structuredOutput: valOutput, textOutput: valText } = await withRateLimitRetry(async () => {
|
||||
validatorToolCalls = 0;
|
||||
|
||||
// Use streamText — Codex endpoint only supports streaming.
|
||||
// Output.object() generates the validation array (wrapped in { validations: [...] }) as a final step.
|
||||
const stream = streamText({
|
||||
model: client.model,
|
||||
system: genOptions.system,
|
||||
messages: [{ role: 'user' as const, content: userMessage }],
|
||||
tools,
|
||||
stopWhen: stepCountIs(150),
|
||||
output: Output.object({ schema: FindingValidationsOutputSchema }),
|
||||
abortSignal,
|
||||
...(genOptions.providerOptions ? { providerOptions: genOptions.providerOptions } : {}),
|
||||
onStepFinish: ({ toolCalls }) => {
|
||||
if (toolCalls && toolCalls.length > 0) {
|
||||
validatorToolCalls += toolCalls.length;
|
||||
this.reportProgress({
|
||||
phase: 'validation',
|
||||
progress: 75,
|
||||
message: `[FindingValidator] Examining code: ${toolCalls.map((tc) => tc.toolName).join(', ')}`,
|
||||
prNumber: context.prNumber,
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Consume stream before reading output
|
||||
for await (const _part of stream.fullStream) { /* consume */ }
|
||||
|
||||
return { structuredOutput: await stream.output, textOutput: await stream.text };
|
||||
}, {
|
||||
signal: abortSignal,
|
||||
onWaiting: (waitMs) => this.reportProgress({
|
||||
phase: 'validation',
|
||||
progress: 72,
|
||||
message: `[FindingValidator] Rate limited. Waiting ${formatWaitDuration(waitMs)}...`,
|
||||
prNumber: context.prNumber,
|
||||
}),
|
||||
});
|
||||
|
||||
// Consume stream before reading output
|
||||
for await (const _part of stream.fullStream) { /* consume */ }
|
||||
|
||||
// Use structured output if available, fall back to text parsing
|
||||
const structuredOutput = await stream.output;
|
||||
let rawValidations: Array<{ findingId: string; validationStatus: string; explanation: string }>;
|
||||
if (structuredOutput) {
|
||||
rawValidations = structuredOutput.validations;
|
||||
if (valOutput) {
|
||||
rawValidations = valOutput.validations;
|
||||
} else {
|
||||
const text = await stream.text;
|
||||
const parsed = parseLLMJson(text, FindingValidationArraySchema);
|
||||
const parsed = parseLLMJson(valText, FindingValidationArraySchema);
|
||||
if (!parsed || !Array.isArray(parsed) || parsed.length === 0) {
|
||||
return findings; // Fail-safe: keep all findings
|
||||
}
|
||||
@@ -913,28 +942,38 @@ Validate each finding by reading the actual code at the specified file and line.
|
||||
};
|
||||
|
||||
try {
|
||||
// Use streamText — Codex endpoint only supports streaming.
|
||||
// Output.object() generates the structured verdict as a final step.
|
||||
const stream = streamText({
|
||||
model: client.model,
|
||||
system: genOptions.system,
|
||||
prompt,
|
||||
output: Output.object({ schema: SynthesisResultOutputSchema }),
|
||||
abortSignal,
|
||||
...(genOptions.providerOptions ? { providerOptions: genOptions.providerOptions } : {}),
|
||||
const { structuredOutput: synthOutput, textOutput: synthText } = await withRateLimitRetry(async () => {
|
||||
// Use streamText — Codex endpoint only supports streaming.
|
||||
// Output.object() generates the structured verdict as a final step.
|
||||
const stream = streamText({
|
||||
model: client.model,
|
||||
system: genOptions.system,
|
||||
prompt,
|
||||
output: Output.object({ schema: SynthesisResultOutputSchema }),
|
||||
abortSignal,
|
||||
...(genOptions.providerOptions ? { providerOptions: genOptions.providerOptions } : {}),
|
||||
});
|
||||
|
||||
// Consume stream before reading output
|
||||
for await (const _part of stream.fullStream) { /* consume */ }
|
||||
|
||||
return { structuredOutput: await stream.output, textOutput: await stream.text };
|
||||
}, {
|
||||
signal: abortSignal,
|
||||
onWaiting: (waitMs) => this.reportProgress({
|
||||
phase: 'synthesizing',
|
||||
progress: 62,
|
||||
message: `[ParallelOrchestrator] Rate limited during synthesis. Waiting ${formatWaitDuration(waitMs)}...`,
|
||||
prNumber: context.prNumber,
|
||||
}),
|
||||
});
|
||||
|
||||
// Consume stream before reading output
|
||||
for await (const _part of stream.fullStream) { /* consume */ }
|
||||
|
||||
// Use structured output if available, fall back to text parsing
|
||||
const structuredOutput = await stream.output;
|
||||
let data: { verdict: string; verdictReasoning: string; removedFindingIds: string[] } | null;
|
||||
if (structuredOutput) {
|
||||
data = structuredOutput;
|
||||
if (synthOutput) {
|
||||
data = synthOutput;
|
||||
} else {
|
||||
const text = await stream.text;
|
||||
data = parseLLMJson(text, SynthesisResultSchema);
|
||||
data = parseLLMJson(synthText, SynthesisResultSchema);
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import { z } from 'zod';
|
||||
|
||||
import { createSimpleClient } from '../../client/factory';
|
||||
import type { ModelShorthand, ThinkingLevel } from '../../config/types';
|
||||
import { withRateLimitRetry } from '../../session/rate-limit-retry';
|
||||
import { parseLLMJson } from '../../schema/structured-output';
|
||||
import {
|
||||
ScanResultSchema,
|
||||
@@ -518,24 +519,24 @@ ${diff}
|
||||
});
|
||||
|
||||
if (reviewPass === ReviewPass.QUICK_SCAN) {
|
||||
const result = await generateText({
|
||||
const result = await withRateLimitRetry(() => generateText({
|
||||
model: client.model,
|
||||
system: client.systemPrompt,
|
||||
prompt: fullPrompt,
|
||||
output: Output.object({ schema: ScanResultOutputSchema }),
|
||||
});
|
||||
}));
|
||||
if (result.output) {
|
||||
return result.output as ScanResult;
|
||||
}
|
||||
return parseScanResult(result.text);
|
||||
}
|
||||
|
||||
const result = await generateText({
|
||||
const result = await withRateLimitRetry(() => generateText({
|
||||
model: client.model,
|
||||
system: client.systemPrompt,
|
||||
prompt: fullPrompt,
|
||||
output: Output.object({ schema: ReviewFindingsOutputSchema }),
|
||||
});
|
||||
}));
|
||||
if (result.output) {
|
||||
return result.output.findings as PRReviewFinding[];
|
||||
}
|
||||
@@ -560,12 +561,12 @@ async function runStructuralPass(
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await generateText({
|
||||
const result = await withRateLimitRetry(() => generateText({
|
||||
model: client.model,
|
||||
system: client.systemPrompt,
|
||||
prompt: fullPrompt,
|
||||
output: Output.object({ schema: StructuralIssuesOutputSchema }),
|
||||
});
|
||||
}));
|
||||
if (result.output) {
|
||||
return result.output.issues as StructuralIssue[];
|
||||
}
|
||||
@@ -596,12 +597,12 @@ async function runAITriagePass(
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await generateText({
|
||||
const result = await withRateLimitRetry(() => generateText({
|
||||
model: client.model,
|
||||
system: client.systemPrompt,
|
||||
prompt: fullPrompt,
|
||||
output: Output.object({ schema: AICommentTriagesOutputSchema }),
|
||||
});
|
||||
}));
|
||||
if (result.output) {
|
||||
return result.output.triages as AICommentTriage[];
|
||||
}
|
||||
|
||||
@@ -19,6 +19,8 @@ import { buildToolRegistry } from '../tools/build-registry';
|
||||
import type { ToolContext } from '../tools/types';
|
||||
import type { ModelShorthand, ThinkingLevel } from '../config/types';
|
||||
import type { SecurityProfile } from '../security/bash-validator';
|
||||
import { withRateLimitRetry } from '../session/rate-limit-retry';
|
||||
import { formatWaitDuration } from '../session/rate-limit-wait';
|
||||
|
||||
// =============================================================================
|
||||
// Constants
|
||||
@@ -97,6 +99,7 @@ export type IdeationStreamCallback = (event: IdeationStreamEvent) => void;
|
||||
export type IdeationStreamEvent =
|
||||
| { type: 'text-delta'; text: string }
|
||||
| { type: 'tool-use'; name: string }
|
||||
| { type: 'status'; text: string }
|
||||
| { type: 'error'; error: string };
|
||||
|
||||
// =============================================================================
|
||||
@@ -186,42 +189,48 @@ export async function runIdeation(
|
||||
const userPrompt = `Analyze the project at ${projectDir} and generate up to ${maxIdeasPerType} ${ideationType.replace(/_/g, ' ')} ideas. Use the available tools to explore the codebase, then write your findings as a JSON file to the output directory.`;
|
||||
|
||||
try {
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodex ? undefined : prompt,
|
||||
prompt: userPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodex ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: prompt,
|
||||
store: false,
|
||||
await withRateLimitRetry(async () => {
|
||||
responseText = '';
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodex ? undefined : prompt,
|
||||
prompt: userPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodex ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: prompt,
|
||||
store: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
} : {}),
|
||||
});
|
||||
} : {}),
|
||||
});
|
||||
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta': {
|
||||
responseText += part.text;
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
}
|
||||
case 'tool-call': {
|
||||
onStream?.({ type: 'tool-use', name: part.toolName });
|
||||
break;
|
||||
}
|
||||
case 'error': {
|
||||
const errorMsg =
|
||||
part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta': {
|
||||
responseText += part.text;
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
}
|
||||
case 'tool-call': {
|
||||
onStream?.({ type: 'tool-use', name: part.toolName });
|
||||
break;
|
||||
}
|
||||
case 'error': {
|
||||
const errorMsg =
|
||||
part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
signal: abortSignal,
|
||||
onWaiting: (waitMs) => onStream?.({ type: 'status', text: `Rate limited. Waiting ${formatWaitDuration(waitMs)}...` }),
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
|
||||
@@ -15,6 +15,9 @@ import { streamText, stepCountIs } from 'ai';
|
||||
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { withRateLimitRetry } from '../session/rate-limit-retry';
|
||||
import { formatWaitDuration } from '../session/rate-limit-wait';
|
||||
|
||||
import { createSimpleClient } from '../client/factory';
|
||||
import { buildToolRegistry } from '../tools/build-registry';
|
||||
import type { ToolContext } from '../tools/types';
|
||||
@@ -85,6 +88,7 @@ export type InsightsStreamEvent =
|
||||
| { type: 'text-delta'; text: string }
|
||||
| { type: 'tool-start'; name: string; input: string }
|
||||
| { type: 'tool-end'; name: string }
|
||||
| { type: 'status'; text: string }
|
||||
| { type: 'error'; error: string };
|
||||
|
||||
// =============================================================================
|
||||
@@ -272,48 +276,59 @@ export async function runInsightsQuery(
|
||||
const isCodexInsights = insightsModelId?.includes('codex') ?? false;
|
||||
|
||||
try {
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodexInsights ? undefined : client.systemPrompt,
|
||||
prompt: fullPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodexInsights ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: client.systemPrompt,
|
||||
store: false,
|
||||
},
|
||||
},
|
||||
} : {}),
|
||||
});
|
||||
await withRateLimitRetry(
|
||||
async () => {
|
||||
responseText = '';
|
||||
toolCalls.length = 0;
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodexInsights ? undefined : client.systemPrompt,
|
||||
prompt: fullPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodexInsights ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: client.systemPrompt,
|
||||
store: false,
|
||||
},
|
||||
},
|
||||
} : {}),
|
||||
});
|
||||
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta': {
|
||||
responseText += part.text;
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta': {
|
||||
responseText += part.text;
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
}
|
||||
case 'tool-call': {
|
||||
const args = 'input' in part ? (part.input as Record<string, unknown>) : {};
|
||||
const input = extractToolInput(args);
|
||||
toolCalls.push({ name: part.toolName, input });
|
||||
onStream?.({ type: 'tool-start', name: part.toolName, input });
|
||||
break;
|
||||
}
|
||||
case 'tool-result': {
|
||||
onStream?.({ type: 'tool-end', name: part.toolName });
|
||||
break;
|
||||
}
|
||||
case 'error': {
|
||||
const errorMsg = part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
case 'tool-call': {
|
||||
const args = 'input' in part ? (part.input as Record<string, unknown>) : {};
|
||||
const input = extractToolInput(args);
|
||||
toolCalls.push({ name: part.toolName, input });
|
||||
onStream?.({ type: 'tool-start', name: part.toolName, input });
|
||||
break;
|
||||
}
|
||||
case 'tool-result': {
|
||||
onStream?.({ type: 'tool-end', name: part.toolName });
|
||||
break;
|
||||
}
|
||||
case 'error': {
|
||||
const errorMsg = part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
signal: abortSignal,
|
||||
onWaiting: (waitMs) =>
|
||||
onStream?.({ type: 'status', text: `Rate limited. Waiting ${formatWaitDuration(waitMs)}...` }),
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
|
||||
@@ -21,6 +21,8 @@ import type { ModelShorthand, ThinkingLevel } from '../config/types';
|
||||
import type { SecurityProfile } from '../security/bash-validator';
|
||||
import { safeParseJson } from '../../utils/json-repair';
|
||||
import { tryLoadPrompt } from '../prompts/prompt-loader';
|
||||
import { withRateLimitRetry } from '../session/rate-limit-retry';
|
||||
import { formatWaitDuration } from '../session/rate-limit-wait';
|
||||
|
||||
// =============================================================================
|
||||
// Constants
|
||||
@@ -86,6 +88,7 @@ export type RoadmapStreamEvent =
|
||||
| { type: 'phase-complete'; phase: string; success: boolean }
|
||||
| { type: 'text-delta'; text: string }
|
||||
| { type: 'tool-use'; name: string }
|
||||
| { type: 'status'; text: string }
|
||||
| { type: 'error'; error: string };
|
||||
|
||||
// =============================================================================
|
||||
@@ -145,38 +148,43 @@ Do NOT ask questions. Make educated inferences and create the file.`;
|
||||
const discoveryUserPrompt = 'Analyze the project and create the discovery document. Use the available tools to explore the codebase, then write your findings as JSON to the output file specified in the context above.';
|
||||
|
||||
try {
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodexDiscovery ? undefined : prompt,
|
||||
prompt: discoveryUserPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodexDiscovery ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: prompt,
|
||||
store: false,
|
||||
await withRateLimitRetry(async () => {
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodexDiscovery ? undefined : prompt,
|
||||
prompt: discoveryUserPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodexDiscovery ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: prompt,
|
||||
store: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
} : {}),
|
||||
});
|
||||
} : {}),
|
||||
});
|
||||
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta':
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
case 'tool-call':
|
||||
onStream?.({ type: 'tool-use', name: part.toolName });
|
||||
break;
|
||||
case 'error': {
|
||||
const errorMsg = part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta':
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
case 'tool-call':
|
||||
onStream?.({ type: 'tool-use', name: part.toolName });
|
||||
break;
|
||||
case 'error': {
|
||||
const errorMsg = part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
signal: abortSignal,
|
||||
onWaiting: (waitMs) => onStream?.({ type: 'status', text: `Rate limited. Waiting ${formatWaitDuration(waitMs)}...` }),
|
||||
});
|
||||
|
||||
// Validate output
|
||||
if (existsSync(discoveryFile)) {
|
||||
@@ -277,38 +285,43 @@ The JSON must contain: vision, target_audience (object with "primary" key), phas
|
||||
const featuresUserPrompt = 'Read the discovery data and generate a complete roadmap with prioritized features. Write the roadmap JSON to the output file specified in the context above.';
|
||||
|
||||
try {
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodexFeatures ? undefined : prompt,
|
||||
prompt: featuresUserPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodexFeatures ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: prompt,
|
||||
store: false,
|
||||
await withRateLimitRetry(async () => {
|
||||
const result = streamText({
|
||||
model: client.model,
|
||||
system: isCodexFeatures ? undefined : prompt,
|
||||
prompt: featuresUserPrompt,
|
||||
tools: client.tools,
|
||||
stopWhen: stepCountIs(client.maxSteps),
|
||||
abortSignal,
|
||||
...(isCodexFeatures ? {
|
||||
providerOptions: {
|
||||
openai: {
|
||||
instructions: prompt,
|
||||
store: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
} : {}),
|
||||
});
|
||||
} : {}),
|
||||
});
|
||||
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta':
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
case 'tool-call':
|
||||
onStream?.({ type: 'tool-use', name: part.toolName });
|
||||
break;
|
||||
case 'error': {
|
||||
const errorMsg = part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case 'text-delta':
|
||||
onStream?.({ type: 'text-delta', text: part.text });
|
||||
break;
|
||||
case 'tool-call':
|
||||
onStream?.({ type: 'tool-use', name: part.toolName });
|
||||
break;
|
||||
case 'error': {
|
||||
const errorMsg = part.error instanceof Error ? part.error.message : String(part.error);
|
||||
onStream?.({ type: 'error', error: errorMsg });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
signal: abortSignal,
|
||||
onWaiting: (waitMs) => onStream?.({ type: 'status', text: `Rate limited. Waiting ${formatWaitDuration(waitMs)}...` }),
|
||||
});
|
||||
|
||||
// Validate and merge — read/write through fd to avoid TOCTOU
|
||||
let roadmapRaw: string | null = null;
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
* - generic: Everything else
|
||||
*/
|
||||
|
||||
import { APICallError } from '@ai-sdk/provider';
|
||||
|
||||
import type { SessionError, SessionOutcome } from './types';
|
||||
|
||||
// =============================================================================
|
||||
@@ -96,6 +98,7 @@ export function isBillingError(error: unknown): boolean {
|
||||
*/
|
||||
export function isRateLimitError(error: unknown): boolean {
|
||||
if (isBillingError(error)) return false;
|
||||
if (APICallError.isInstance(error) && error.statusCode === 429) return true;
|
||||
const errorStr = errorToString(error);
|
||||
if (WORD_BOUNDARY_429.test(errorStr)) return true;
|
||||
return RATE_LIMIT_PATTERNS.some((p) => errorStr.includes(p));
|
||||
@@ -105,6 +108,7 @@ export function isRateLimitError(error: unknown): boolean {
|
||||
* Check if an error is an authentication error (401 or similar).
|
||||
*/
|
||||
export function isAuthenticationError(error: unknown): boolean {
|
||||
if (APICallError.isInstance(error) && error.statusCode === 401) return true;
|
||||
const errorStr = errorToString(error);
|
||||
if (WORD_BOUNDARY_401.test(errorStr)) return true;
|
||||
return AUTH_PATTERNS.some((p) => errorStr.includes(p));
|
||||
@@ -114,6 +118,14 @@ export function isAuthenticationError(error: unknown): boolean {
|
||||
* Check if an error is a 400 tool concurrency error from Claude API.
|
||||
*/
|
||||
export function isToolConcurrencyError(error: unknown): boolean {
|
||||
if (APICallError.isInstance(error) && error.statusCode === 400) {
|
||||
const errorStr = errorToString(error);
|
||||
return (
|
||||
(errorStr.includes('tool') && errorStr.includes('concurrency')) ||
|
||||
errorStr.includes('too many tools') ||
|
||||
errorStr.includes('concurrent tool')
|
||||
);
|
||||
}
|
||||
const errorStr = errorToString(error);
|
||||
return (
|
||||
/\b400\b/.test(errorStr) &&
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* Rate Limit Retry Wrapper
|
||||
* ========================
|
||||
*
|
||||
* Wraps AI calls (streamText, generateText) with rate-limit-aware retry logic.
|
||||
* When all accounts are exhausted and the auto-swap middleware can't recover,
|
||||
* this utility waits for the rate limit to reset and retries the operation.
|
||||
*
|
||||
* Usage:
|
||||
* ```ts
|
||||
* const result = await withRateLimitRetry(
|
||||
* () => generateText({ model, prompt, ... }),
|
||||
* { signal: abortSignal, onWaiting: (ms) => onStream?.({ type: 'status', text: `Rate limited, waiting...` }) }
|
||||
* );
|
||||
* ```
|
||||
*/
|
||||
|
||||
import { isRateLimitError } from './error-classifier';
|
||||
import { extractWaitDuration, formatWaitDuration, sleepWithAbort } from './rate-limit-wait';
|
||||
|
||||
/** Maximum retries after rate limit waits */
|
||||
const MAX_RATE_LIMIT_RETRIES = 3;
|
||||
|
||||
export interface RateLimitRetryOptions {
|
||||
/** Abort signal to cancel the wait */
|
||||
signal?: AbortSignal;
|
||||
/** Maximum retries (default: 3) */
|
||||
maxRetries?: number;
|
||||
/** Called when entering a rate-limit wait, with the wait duration in ms */
|
||||
onWaiting?: (waitMs: number, retryCount: number) => void;
|
||||
/** Called when a wait completes and the operation is being retried */
|
||||
onRetry?: (retryCount: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute an async operation with rate-limit-aware retry.
|
||||
*
|
||||
* If the operation throws a rate limit error (after the auto-swap middleware
|
||||
* has already exhausted its options), this waits for the reset and retries.
|
||||
* Weekly limits (too long to wait) are NOT retried — they throw immediately.
|
||||
*/
|
||||
export async function withRateLimitRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options?: RateLimitRetryOptions,
|
||||
): Promise<T> {
|
||||
const maxRetries = options?.maxRetries ?? MAX_RATE_LIMIT_RETRIES;
|
||||
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
if (!isRateLimitError(error)) {
|
||||
console.log('[AutoSwap:Retry] Non-rate-limit error — not retrying:', error instanceof Error ? error.message : String(error));
|
||||
throw error;
|
||||
}
|
||||
if (attempt >= maxRetries) {
|
||||
console.log('[AutoSwap:Retry] Max retries reached (' + maxRetries + ') — giving up');
|
||||
throw error;
|
||||
}
|
||||
|
||||
const waitInfo = extractWaitDuration(error);
|
||||
// Weekly limits or unextractable — don't wait
|
||||
if (!waitInfo) {
|
||||
console.log('[AutoSwap:Retry] Cannot extract wait duration — not retrying (weekly limit?)');
|
||||
throw error;
|
||||
}
|
||||
|
||||
console.log('[AutoSwap:Retry] Rate limit hit (attempt', attempt + 1 + '/' + maxRetries + '). Waiting', formatWaitDuration(waitInfo.waitMs), '(type:', waitInfo.limitType + ')');
|
||||
options?.onWaiting?.(waitInfo.waitMs, attempt + 1);
|
||||
|
||||
const completed = await sleepWithAbort(waitInfo.waitMs, options?.signal);
|
||||
if (!completed) {
|
||||
console.log('[AutoSwap:Retry] Wait aborted — not retrying');
|
||||
throw error;
|
||||
}
|
||||
|
||||
console.log('[AutoSwap:Retry] Wait complete — retrying (attempt', attempt + 2 + ')');
|
||||
options?.onRetry?.(attempt + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* Rate Limit Wait-and-Retry
|
||||
* =========================
|
||||
*
|
||||
* Shared utility for extracting wait duration from rate limit errors
|
||||
* and sleeping with cancellation support.
|
||||
*
|
||||
* Used by both:
|
||||
* - Session runner (task pipeline: planner, coder, QA)
|
||||
* - Auto-swap middleware (standalone runners: insights, roadmap, PR review, etc.)
|
||||
*
|
||||
* When a 429 occurs and no swap target is available, this module extracts
|
||||
* how long to wait from the error and sleeps until the rate limit resets.
|
||||
*/
|
||||
|
||||
import { APICallError } from '@ai-sdk/provider';
|
||||
import { parseResetTime, classifyRateLimitType } from '../../claude-profile/usage-parser';
|
||||
|
||||
// =============================================================================
|
||||
// Constants
|
||||
// =============================================================================
|
||||
|
||||
/** Maximum time to auto-wait for a rate limit reset (30 minutes) */
|
||||
const MAX_RATE_LIMIT_WAIT_MS = 30 * 60 * 1000;
|
||||
|
||||
/** Default wait time when no reset info is available (5 minutes) */
|
||||
const DEFAULT_WAIT_MS = 5 * 60 * 1000;
|
||||
|
||||
/** Minimum wait time to avoid tight retry loops (30 seconds) */
|
||||
const MIN_WAIT_MS = 30 * 1000;
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
// =============================================================================
|
||||
|
||||
export interface WaitDurationInfo {
|
||||
/** How long to wait in milliseconds */
|
||||
waitMs: number;
|
||||
/** When the rate limit resets (if known) */
|
||||
resetAt: Date | null;
|
||||
/** Whether this is a session or weekly limit */
|
||||
limitType: 'session' | 'weekly';
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Wait Duration Extraction
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Extract wait duration from a rate limit error.
|
||||
*
|
||||
* Priority order:
|
||||
* 1. `Retry-After` header (seconds or HTTP-date)
|
||||
* 2. Reset time string from error message (e.g., "resets Dec 17 at 6am")
|
||||
* 3. Fallback: 5 minutes (only when no extraction attempted — i.e., no reset info found)
|
||||
*
|
||||
* Returns null for weekly limits (too long to auto-wait) or when message extraction
|
||||
* explicitly detected a weekly limit and declined to provide a wait time.
|
||||
*/
|
||||
export function extractWaitDuration(error: unknown): WaitDurationInfo | null {
|
||||
// Try Retry-After header from APICallError
|
||||
const headerWait = extractFromRetryAfterHeader(error);
|
||||
if (headerWait) return headerWait;
|
||||
|
||||
// Try reset time from error message
|
||||
// extractFromErrorMessage returns null for weekly limits — don't fall back in that case
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const hasResetPattern = /resets?\s+/i.test(message);
|
||||
if (hasResetPattern) {
|
||||
// Extraction was attempted; trust its result (null = weekly limit, don't wait)
|
||||
return extractFromErrorMessage(error);
|
||||
}
|
||||
|
||||
// No reset info found at all — fall back to 5 minutes (safe default for session limits)
|
||||
return {
|
||||
waitMs: DEFAULT_WAIT_MS,
|
||||
resetAt: new Date(Date.now() + DEFAULT_WAIT_MS),
|
||||
limitType: 'session',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract wait duration from Retry-After response header.
|
||||
* Supports both seconds (e.g., "120") and HTTP-date (e.g., "Thu, 01 Dec 2025 16:00:00 GMT").
|
||||
*/
|
||||
function extractFromRetryAfterHeader(error: unknown): WaitDurationInfo | null {
|
||||
if (!APICallError.isInstance(error)) return null;
|
||||
const retryAfter = error.responseHeaders?.['retry-after'];
|
||||
if (!retryAfter) return null;
|
||||
|
||||
// Try as seconds
|
||||
const seconds = Number(retryAfter);
|
||||
if (!Number.isNaN(seconds) && seconds > 0) {
|
||||
const waitMs = clampWaitMs(seconds * 1000);
|
||||
return {
|
||||
waitMs,
|
||||
resetAt: new Date(Date.now() + waitMs),
|
||||
limitType: 'session',
|
||||
};
|
||||
}
|
||||
|
||||
// Try as HTTP-date
|
||||
const date = new Date(retryAfter);
|
||||
if (!Number.isNaN(date.getTime())) {
|
||||
const waitMs = clampWaitMs(date.getTime() - Date.now());
|
||||
return {
|
||||
waitMs,
|
||||
resetAt: date,
|
||||
limitType: 'session',
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract wait duration from error message patterns.
|
||||
* Looks for reset time strings like "resets Dec 17 at 6am (Europe/Oslo)".
|
||||
*/
|
||||
function extractFromErrorMessage(error: unknown): WaitDurationInfo | null {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
// Look for reset time pattern: "resets <datetime>" or "Resets <datetime>"
|
||||
const resetMatch = message.match(/resets?\s+(.+?)(?:\s*\.|$|\n)/i);
|
||||
if (!resetMatch) return null;
|
||||
|
||||
const resetTimeStr = resetMatch[1].trim();
|
||||
const limitType = classifyRateLimitType(resetTimeStr);
|
||||
|
||||
// Don't auto-wait for weekly limits — too long
|
||||
if (limitType === 'weekly') return null;
|
||||
|
||||
const resetAt = parseResetTime(resetTimeStr);
|
||||
const waitMs = clampWaitMs(resetAt.getTime() - Date.now());
|
||||
|
||||
return { waitMs, resetAt, limitType };
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Sleep with Abort
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Sleep for the given duration, but cancel early if the abort signal fires.
|
||||
* Returns true if sleep completed normally, false if aborted.
|
||||
*/
|
||||
export function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<boolean> {
|
||||
if (signal?.aborted) return Promise.resolve(false);
|
||||
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
resolve(true);
|
||||
}, ms);
|
||||
|
||||
function onAbort() {
|
||||
clearTimeout(timer);
|
||||
resolve(false);
|
||||
}
|
||||
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Formatting
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Format a duration in milliseconds to a human-readable string.
|
||||
*/
|
||||
export function formatWaitDuration(ms: number): string {
|
||||
const totalSeconds = Math.ceil(ms / 1000);
|
||||
if (totalSeconds < 60) return `${totalSeconds}s`;
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
if (seconds === 0) return `${minutes}m`;
|
||||
return `${minutes}m ${seconds}s`;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Helpers
|
||||
// =============================================================================
|
||||
|
||||
function clampWaitMs(ms: number): number {
|
||||
return Math.max(MIN_WAIT_MS, Math.min(ms, MAX_RATE_LIMIT_WAIT_MS));
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import { buildThinkingProviderOptions } from '../config/types';
|
||||
import { createStreamHandler } from './stream-handler';
|
||||
import type { FullStreamPart } from './stream-handler';
|
||||
import { classifyError, isAuthenticationError, isRateLimitError } from './error-classifier';
|
||||
import { extractWaitDuration, sleepWithAbort, formatWaitDuration } from './rate-limit-wait';
|
||||
import { ProgressTracker } from './progress-tracker';
|
||||
import type {
|
||||
SessionConfig,
|
||||
@@ -196,7 +197,19 @@ export async function runAgentSession(
|
||||
activeAccountId = newAuth.accountId;
|
||||
continue;
|
||||
}
|
||||
// No more accounts available — fall through to legacy retry
|
||||
// No more accounts available — try wait-and-retry for session limits
|
||||
if (isRateLimitError(error)) {
|
||||
const waitInfo = extractWaitDuration(error);
|
||||
if (waitInfo) {
|
||||
onEvent?.({
|
||||
type: 'status',
|
||||
text: `Rate limited. Waiting ${formatWaitDuration(waitInfo.waitMs)} for reset...`,
|
||||
});
|
||||
const completed = await sleepWithAbort(waitInfo.waitMs, activeConfig.abortSignal);
|
||||
if (completed) continue; // Retry after wait
|
||||
// Aborted during wait — fall through to error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy auth refresh (single-provider token refresh)
|
||||
@@ -363,6 +376,7 @@ async function executeStream(
|
||||
|
||||
const result = streamText({
|
||||
model: config.model,
|
||||
maxRetries: 0, // Prevent AI SDK from retrying against same rate-limited account — our catch block handles retry with account switching
|
||||
system: isCodex ? undefined : config.systemPrompt,
|
||||
messages: aiMessages,
|
||||
tools: tools ?? {},
|
||||
|
||||
@@ -170,7 +170,8 @@ export type StreamEvent =
|
||||
| ToolResultEvent
|
||||
| StepFinishEvent
|
||||
| ErrorEvent
|
||||
| UsageUpdateEvent;
|
||||
| UsageUpdateEvent
|
||||
| StatusEvent;
|
||||
|
||||
/** Incremental text output from the model */
|
||||
export interface TextDeltaEvent {
|
||||
@@ -221,6 +222,12 @@ export interface UsageUpdateEvent {
|
||||
usage: TokenUsage;
|
||||
}
|
||||
|
||||
/** Status update (e.g., waiting for rate limit reset) */
|
||||
export interface StatusEvent {
|
||||
type: 'status';
|
||||
text: string;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Progress State
|
||||
// =============================================================================
|
||||
|
||||
@@ -214,6 +214,7 @@ export class UsageMonitor extends EventEmitter {
|
||||
private currentUsage: ClaudeUsageSnapshot | null = null;
|
||||
private currentUsageProfileId: string | null = null; // Track which profile's usage is in currentUsage
|
||||
private isChecking = false;
|
||||
private pendingCheckAfterCurrent = false;
|
||||
|
||||
// Per-profile API failure tracking with cooldown-based retry
|
||||
// Map<profileId, lastFailureTimestamp> - stores when API last failed for this profile
|
||||
@@ -231,7 +232,7 @@ export class UsageMonitor extends EventEmitter {
|
||||
// Cache for all profiles' usage data
|
||||
// Map<profileId, { usage: ProfileUsageSummary, fetchedAt: number }>
|
||||
private allProfilesUsageCache: Map<string, { usage: ProfileUsageSummary; fetchedAt: number }> = new Map();
|
||||
private static PROFILE_USAGE_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes cache for inactive profiles
|
||||
private static PROFILE_USAGE_CACHE_TTL_MS = 10 * 60 * 1000; // 10 minutes cache for inactive profiles (endpoint is fragile, reduce load)
|
||||
|
||||
// Request coalescing: track in-flight getAllProfilesUsage() promise to avoid parallel duplicate fetches
|
||||
private allProfilesUsageInflight: Promise<AllProfilesUsage | null> | null = null;
|
||||
@@ -241,7 +242,7 @@ export class UsageMonitor extends EventEmitter {
|
||||
|
||||
// Rate-limit (429) tracking: separate from general API failures, uses longer cooldown
|
||||
private rateLimitedProfiles: Map<string, number> = new Map(); // profileId -> 429 timestamp
|
||||
private static RATE_LIMIT_COOLDOWN_MS = 10 * 60 * 1000; // 10 minutes cooldown for 429s
|
||||
private static RATE_LIMIT_COOLDOWN_MS = 15 * 60 * 1000; // 15 minutes cooldown for 429s (endpoint may not recover quickly)
|
||||
|
||||
// Debug flag for verbose logging
|
||||
private readonly isDebug = process.env.DEBUG === 'true';
|
||||
@@ -292,7 +293,11 @@ export class UsageMonitor extends EventEmitter {
|
||||
* Note: Usage monitoring always runs to display the usage badge.
|
||||
* Proactive account swapping only occurs if enabled in settings.
|
||||
*
|
||||
* Update interval: 60 seconds (60000ms) for active profile; inactive profiles every 5 minutes (adaptive: 60s when usage is high)
|
||||
* Update interval: 5 minutes (300000ms) for active profile; inactive profiles every 10 minutes (adaptive: 2min when usage is high)
|
||||
*
|
||||
* Note: The /api/oauth/usage endpoint is fragile and undocumented. 60s polling triggers 429s
|
||||
* (see anthropics/claude-code issues #30930, #31021, #31055, #31637, #32503).
|
||||
* 5-minute intervals align with practical safe limits for this endpoint.
|
||||
*/
|
||||
start(): void {
|
||||
if (this.intervalId) {
|
||||
@@ -302,11 +307,16 @@ export class UsageMonitor extends EventEmitter {
|
||||
|
||||
const profileManager = getClaudeProfileManager();
|
||||
const settings = profileManager.getAutoSwitchSettings();
|
||||
const interval = settings.usageCheckInterval || 60000; // 60 seconds for active profile polling
|
||||
const interval = settings.usageCheckInterval || 300000; // 5 minutes for active profile polling (endpoint is fragile)
|
||||
|
||||
this.debugLog('[UsageMonitor] Starting with interval: ' + interval + ' ms (60-second updates for active profile usage stats)');
|
||||
this.debugLog('[UsageMonitor] Starting with interval: ' + interval + ' ms (5-minute updates for active profile usage stats)');
|
||||
|
||||
// Check immediately
|
||||
// Fetch all profiles usage immediately on startup so the UI shows real data.
|
||||
// This runs independently of checkUsageAndSwap — even if the active profile's
|
||||
// fetch fails (e.g., 429), inactive profiles still get fetched and displayed.
|
||||
this.fetchAllProfilesOnStartup();
|
||||
|
||||
// Check immediately (handles proactive swap for active profile)
|
||||
this.checkUsageAndSwap();
|
||||
|
||||
// Then check periodically
|
||||
@@ -326,6 +336,43 @@ export class UsageMonitor extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all profiles' usage on startup so the UI shows real data immediately.
|
||||
* Runs asynchronously — doesn't block app startup. Errors are non-fatal.
|
||||
*
|
||||
* We wait briefly for checkUsageAndSwap() to populate currentUsage (active profile),
|
||||
* then call getAllProfilesUsage() which fetches inactive profiles and combines everything.
|
||||
* If the active profile fetch fails (e.g., 429), we still call _doGetAllProfilesUsage
|
||||
* directly to populate inactive profile data for the UI.
|
||||
*/
|
||||
private async fetchAllProfilesOnStartup(): Promise<void> {
|
||||
try {
|
||||
// Wait for checkUsageAndSwap() to have a chance to populate currentUsage.
|
||||
// 5s is enough for most API responses; if it fails, we proceed with partial data.
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
|
||||
console.log('[UsageMonitor] Startup: fetching all profiles usage for UI...');
|
||||
|
||||
let allProfilesUsage: AllProfilesUsage | null;
|
||||
if (this.currentUsage) {
|
||||
// Active profile data is available — use the normal path
|
||||
allProfilesUsage = await this.getAllProfilesUsage(true);
|
||||
} else {
|
||||
// Active profile fetch failed (429, etc.) — fetch inactive profiles directly
|
||||
console.log('[UsageMonitor] Startup: active profile usage unavailable, fetching inactive profiles directly');
|
||||
allProfilesUsage = await this._doGetAllProfilesUsage(true);
|
||||
}
|
||||
|
||||
if (allProfilesUsage) {
|
||||
this.emit('all-profiles-usage-updated', allProfilesUsage);
|
||||
console.log('[UsageMonitor] Startup: all profiles usage emitted to UI (' + allProfilesUsage.allProfiles.length + ' profiles)');
|
||||
}
|
||||
} catch (error) {
|
||||
// Non-fatal — the regular polling will pick it up
|
||||
console.log('[UsageMonitor] Startup: failed to fetch all profiles usage (non-fatal):', error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current usage snapshot (for UI indicator)
|
||||
*/
|
||||
@@ -443,6 +490,7 @@ export class UsageMonitor extends EventEmitter {
|
||||
await this.appendZAIAccounts(allProfiles);
|
||||
|
||||
// Return minimal data with auth status - don't return null!
|
||||
console.log('[UsageMonitor] getAllProfilesUsage fast-path (no currentUsage):', allProfiles.map(p => ({ id: p.profileId, name: p.profileName, email: p.profileEmail, active: p.isActive })));
|
||||
return {
|
||||
activeProfile: {
|
||||
profileId: activeProfileId || '',
|
||||
@@ -486,13 +534,14 @@ export class UsageMonitor extends EventEmitter {
|
||||
const profileResults: (ProfileUsageSummary | null)[] = new Array(settings.profiles.length).fill(null);
|
||||
|
||||
// Adaptive cache TTL: when active profile usage is high, refresh inactive profiles more
|
||||
// frequently (every 60s instead of 5min) because we may need to swap soon
|
||||
// frequently (every 2min instead of 10min) because we may need to swap soon.
|
||||
// Note: even in swap-ready mode, we avoid <2min to respect the fragile endpoint.
|
||||
const activeUsageHigh = this.currentUsage
|
||||
? (this.currentUsage.sessionPercent > 80 || this.currentUsage.weeklyPercent > 90)
|
||||
: false;
|
||||
const effectiveCacheTtl = activeUsageHigh
|
||||
? 60 * 1000 // 60s when usage is high (swap-ready mode)
|
||||
: UsageMonitor.PROFILE_USAGE_CACHE_TTL_MS; // 5 min normally
|
||||
? 2 * 60 * 1000 // 2min when usage is high (swap-ready mode)
|
||||
: UsageMonitor.PROFILE_USAGE_CACHE_TTL_MS; // 10 min normally
|
||||
|
||||
for (let i = 0; i < settings.profiles.length; i++) {
|
||||
const profile = settings.profiles[i];
|
||||
@@ -755,9 +804,18 @@ export class UsageMonitor extends EventEmitter {
|
||||
// Sort by availability score (highest first = most available)
|
||||
allProfiles.sort((a, b) => b.availabilityScore - a.availabilityScore);
|
||||
|
||||
// Build active profile data — may be null on startup if the active profile's fetch failed
|
||||
const activeProfile = this.currentUsage ?? {
|
||||
profileId: settings.activeProfileId || '',
|
||||
profileName: settings.profiles.find(p => p.id === settings.activeProfileId)?.name || '',
|
||||
sessionPercent: 0,
|
||||
weeklyPercent: 0,
|
||||
fetchedAt: new Date(),
|
||||
};
|
||||
|
||||
console.log('[UsageMonitor] _doGetAllProfilesUsage complete:', allProfiles.map(p => ({ id: p.profileId, name: p.profileName, session: p.sessionPercent, weekly: p.weeklyPercent, active: p.isActive })));
|
||||
return {
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
activeProfile: this.currentUsage!, // Non-null: _doGetAllProfilesUsage is only called when currentUsage is set
|
||||
activeProfile,
|
||||
allProfiles,
|
||||
fetchedAt: new Date()
|
||||
};
|
||||
@@ -1311,7 +1369,8 @@ export class UsageMonitor extends EventEmitter {
|
||||
*/
|
||||
private async checkUsageAndSwap(): Promise<void> {
|
||||
if (this.isChecking) {
|
||||
return; // Prevent concurrent checks
|
||||
this.pendingCheckAfterCurrent = true; // Re-run after current check completes
|
||||
return;
|
||||
}
|
||||
|
||||
this.isChecking = true;
|
||||
@@ -1404,7 +1463,23 @@ export class UsageMonitor extends EventEmitter {
|
||||
this.traceLog('[UsageMonitor:TRACE] Skipping proactive swap for API profile (only supported for OAuth profiles)');
|
||||
}
|
||||
} catch (error) {
|
||||
// Step 5: Handle auth failures
|
||||
// Step 5a: Handle usage API 429 — trigger proactive swap
|
||||
// A 429 from the usage endpoint means the account is at/near its rate limit.
|
||||
// Instead of sitting idle for 10 minutes, proactively swap to an available account.
|
||||
if (isHttpError(error) && (error as any).isUsageApi429 && profileId && !isAPIProfile) {
|
||||
const profileManager = getClaudeProfileManager();
|
||||
const settings = profileManager.getAutoSwitchSettings();
|
||||
|
||||
if (settings.enabled && settings.proactiveSwapEnabled) {
|
||||
console.log('[UsageMonitor] Usage API returned 429 — treating as threshold exceeded, triggering proactive swap for profile:', profileId);
|
||||
await this.performProactiveSwap(profileId, 'session');
|
||||
return;
|
||||
}
|
||||
console.log('[UsageMonitor] Usage API returned 429 but proactive swap is disabled — backing off silently');
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 5b: Handle auth failures
|
||||
if (isHttpError(error) && (error.statusCode === 401 || error.statusCode === 403)) {
|
||||
if (profileId) {
|
||||
await this.handleAuthFailure(profileId, isAPIProfile);
|
||||
@@ -1415,6 +1490,13 @@ export class UsageMonitor extends EventEmitter {
|
||||
console.error('[UsageMonitor] Check failed:', error);
|
||||
} finally {
|
||||
this.isChecking = false;
|
||||
// If a check was requested while we were busy, run it now
|
||||
if (this.pendingCheckAfterCurrent) {
|
||||
this.pendingCheckAfterCurrent = false;
|
||||
this.checkUsageAndSwap().catch(error => {
|
||||
console.error('[UsageMonitor] Pending check failed:', error);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2203,7 +2285,7 @@ export class UsageMonitor extends EventEmitter {
|
||||
if (response.status === 429) {
|
||||
const now = Date.now();
|
||||
const siblingIds = this.getProfileIdFamily(profileId);
|
||||
console.warn('[UsageMonitor] Rate limited (429) by provider, backing off for 10 minutes:', {
|
||||
console.warn('[UsageMonitor] Rate limited (429) by provider, backing off for 15 minutes:', {
|
||||
provider,
|
||||
endpoint: usageEndpoint,
|
||||
cooldownMs: UsageMonitor.RATE_LIMIT_COOLDOWN_MS,
|
||||
@@ -2212,7 +2294,13 @@ export class UsageMonitor extends EventEmitter {
|
||||
for (const id of siblingIds) {
|
||||
this.rateLimitedProfiles.set(id, now);
|
||||
}
|
||||
return null;
|
||||
// Signal to checkUsageAndSwap that the usage API was 429'd.
|
||||
// A 429 on the usage endpoint is strong evidence the account is at/near
|
||||
// its rate limit — we should proactively swap rather than sit idle for 10 min.
|
||||
const error = new Error('Usage API rate limited (429)');
|
||||
(error as any).statusCode = 429;
|
||||
(error as any).isUsageApi429 = true;
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Check for auth failures via status code (works for all providers)
|
||||
@@ -2354,6 +2442,11 @@ export class UsageMonitor extends EventEmitter {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Re-throw usage API 429 so checkUsageAndSwap can trigger proactive swap
|
||||
if (error?.isUsageApi429) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
console.error('[UsageMonitor] API fetch failed:', error);
|
||||
// Record failure timestamp for cooldown retry (network/other errors)
|
||||
this.apiFailureTimestamps.set(profileId, Date.now());
|
||||
@@ -2653,6 +2746,60 @@ export class UsageMonitor extends EventEmitter {
|
||||
const profileManager = getClaudeProfileManager();
|
||||
const excludeIds = new Set([currentProfileId, ...additionalExclusions]);
|
||||
|
||||
// Also exclude the currently active profile — the swap target must differ from
|
||||
// what the app is already running on, not just from what triggered the swap.
|
||||
const activeProfileId = profileManager.getActiveProfile()?.id;
|
||||
if (activeProfileId) {
|
||||
excludeIds.add(activeProfileId);
|
||||
}
|
||||
|
||||
// Also exclude the linked claudeProfileId when currentProfileId is a provider account ID.
|
||||
// Provider accounts (pa_*) and OAuth profiles (msup, etc.) use different ID formats
|
||||
// but refer to the same underlying account. Without this, the same account would be
|
||||
// picked as the swap target — resulting in a no-op swap.
|
||||
try {
|
||||
const settings = await readSettingsFileAsync();
|
||||
const providerAccounts = (settings?.providerAccounts as ProviderAccount[] | undefined) ?? [];
|
||||
const currentAccount = providerAccounts.find(a => a.id === currentProfileId);
|
||||
if (currentAccount?.claudeProfileId) {
|
||||
excludeIds.add(currentAccount.claudeProfileId);
|
||||
}
|
||||
// Also handle the reverse: if currentProfileId is a claudeProfileId, exclude matching provider accounts
|
||||
for (const account of providerAccounts) {
|
||||
if (account.claudeProfileId === currentProfileId) {
|
||||
excludeIds.add(account.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Exclude all provider accounts that share the same configDir as the current one.
|
||||
// Multiple profiles on the same Anthropic account share the same rate limit —
|
||||
// swapping between them is pointless.
|
||||
const resolvedAccount = currentAccount ?? providerAccounts.find(a => a.claudeProfileId === currentProfileId);
|
||||
if (resolvedAccount) {
|
||||
const currentConfigDir = resolvedAccount.claudeProfileId
|
||||
? profileManager.getProfile(resolvedAccount.claudeProfileId)?.configDir
|
||||
: undefined;
|
||||
if (currentConfigDir) {
|
||||
const allOAuthProfiles = profileManager.getProfilesSortedByAvailability();
|
||||
for (const profile of allOAuthProfiles) {
|
||||
if (profile.configDir === currentConfigDir && !excludeIds.has(profile.id)) {
|
||||
excludeIds.add(profile.id);
|
||||
}
|
||||
}
|
||||
// Also exclude provider accounts linked to those same profiles
|
||||
for (const account of providerAccounts) {
|
||||
if (account.claudeProfileId && excludeIds.has(account.claudeProfileId)) {
|
||||
excludeIds.add(account.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Non-critical — proceed with partial exclusion set
|
||||
}
|
||||
|
||||
console.log('[UsageMonitor] PROACTIVE-SWAP: Exclude set:', Array.from(excludeIds));
|
||||
|
||||
// Get priority order for unified account system
|
||||
const priorityOrder = profileManager.getAccountPriorityOrder();
|
||||
|
||||
@@ -2667,20 +2814,38 @@ export class UsageMonitor extends EventEmitter {
|
||||
|
||||
const unifiedAccounts: UnifiedSwapTarget[] = [];
|
||||
|
||||
// Add OAuth profiles (sorted by availability)
|
||||
// Add OAuth profiles (sorted by availability), filtering out those above threshold
|
||||
const swapSettings = profileManager.getAutoSwitchSettings();
|
||||
const sessionThreshold = swapSettings.sessionThreshold ?? 95;
|
||||
const weeklyThreshold = swapSettings.weeklyThreshold ?? 99;
|
||||
const oauthProfiles = profileManager.getProfilesSortedByAvailability();
|
||||
for (const profile of oauthProfiles) {
|
||||
if (!excludeIds.has(profile.id)) {
|
||||
const unifiedId = `oauth-${profile.id}`;
|
||||
const priorityIndex = priorityOrder.indexOf(unifiedId);
|
||||
unifiedAccounts.push({
|
||||
id: profile.id,
|
||||
unifiedId,
|
||||
name: profile.name,
|
||||
type: 'oauth',
|
||||
priorityIndex: priorityIndex === -1 ? Infinity : priorityIndex
|
||||
});
|
||||
if (excludeIds.has(profile.id)) continue;
|
||||
|
||||
// Skip profiles that are themselves above the swap threshold
|
||||
const sessionPct = profile.usage?.sessionUsagePercent ?? 0;
|
||||
const weeklyPct = profile.usage?.weeklyUsagePercent ?? 0;
|
||||
if (sessionPct >= sessionThreshold || weeklyPct >= weeklyThreshold) {
|
||||
console.log('[UsageMonitor] PROACTIVE-SWAP: Skipping', profile.name, '— above threshold (session:', sessionPct + '%, weekly:', weeklyPct + '%)');
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip rate-limited profiles
|
||||
const rateLimitStatus = isProfileRateLimited(profile);
|
||||
if (rateLimitStatus.limited) {
|
||||
console.log('[UsageMonitor] PROACTIVE-SWAP: Skipping', profile.name, '— rate limited (type:', rateLimitStatus.type + ')');
|
||||
continue;
|
||||
}
|
||||
|
||||
const unifiedId = `oauth-${profile.id}`;
|
||||
const priorityIndex = priorityOrder.indexOf(unifiedId);
|
||||
unifiedAccounts.push({
|
||||
id: profile.id,
|
||||
unifiedId,
|
||||
name: profile.name,
|
||||
type: 'oauth',
|
||||
priorityIndex: priorityIndex === -1 ? Infinity : priorityIndex
|
||||
});
|
||||
}
|
||||
|
||||
// Add API profiles (always considered available since they have unlimited usage)
|
||||
@@ -2759,6 +2924,30 @@ export class UsageMonitor extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
// Update globalPriorityOrder so the UI reflects the new active account.
|
||||
// The UI's "Active Account" is determined by the first entry in globalPriorityOrder.
|
||||
try {
|
||||
const currentSettings = await readSettingsFileAsync();
|
||||
if (currentSettings) {
|
||||
const providerAccounts = (currentSettings.providerAccounts as ProviderAccount[] | undefined) ?? [];
|
||||
const queue = (currentSettings.globalPriorityOrder as string[] | undefined) ?? [];
|
||||
|
||||
// Find the provider account linked to the target OAuth profile
|
||||
const targetProviderAccount = providerAccounts.find(a => a.claudeProfileId === rawProfileId);
|
||||
if (targetProviderAccount && queue.includes(targetProviderAccount.id)) {
|
||||
// Move target to front of queue
|
||||
const newQueue = [targetProviderAccount.id, ...queue.filter(id => id !== targetProviderAccount.id)];
|
||||
currentSettings.globalPriorityOrder = newQueue;
|
||||
await writeSettingsFile(currentSettings);
|
||||
console.log('[UsageMonitor] PROACTIVE-SWAP: Updated globalPriorityOrder — new active:', targetProviderAccount.name, '(' + targetProviderAccount.id + ')');
|
||||
} else {
|
||||
console.log('[UsageMonitor] PROACTIVE-SWAP: Could not find provider account for OAuth profile:', rawProfileId);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[UsageMonitor] PROACTIVE-SWAP: Failed to update globalPriorityOrder:', error);
|
||||
}
|
||||
|
||||
// Get the "from" profile name
|
||||
let fromProfileName: string | undefined;
|
||||
const fromOAuthProfile = profileManager.getProfile(currentProfileId);
|
||||
@@ -2785,10 +2974,11 @@ export class UsageMonitor extends EventEmitter {
|
||||
timestamp: new Date()
|
||||
});
|
||||
|
||||
// Notify UI
|
||||
// Notify UI — shape must match what ProactiveSwapListener expects:
|
||||
// data.fromProfile.name and data.toProfile.name
|
||||
this.emit('show-swap-notification', {
|
||||
fromProfile: fromProfileName,
|
||||
toProfile: bestAccount.name,
|
||||
fromProfile: { id: currentProfileId, name: fromProfileName ?? currentProfileId },
|
||||
toProfile: { id: bestAccount.id, name: bestAccount.name },
|
||||
reason: 'proactive',
|
||||
limitType
|
||||
});
|
||||
|
||||
@@ -135,6 +135,13 @@ export class InsightsExecutor extends EventEmitter {
|
||||
} as InsightsStreamChunk);
|
||||
break;
|
||||
}
|
||||
case 'status': {
|
||||
this.emit('stream-chunk', projectId, {
|
||||
type: 'text',
|
||||
content: event.text,
|
||||
} as InsightsStreamChunk);
|
||||
break;
|
||||
}
|
||||
case 'error': {
|
||||
allOutput = (allOutput + event.error).slice(-10000);
|
||||
this.emit('stream-chunk', projectId, {
|
||||
|
||||
@@ -165,6 +165,7 @@ export async function createSpecForIssue(
|
||||
githubIssueNumber: issueNumber,
|
||||
githubUrl: safeGithubUrl,
|
||||
category,
|
||||
userDescription: safeDescription,
|
||||
// Store baseBranch for worktree creation and QA comparison
|
||||
// This comes from project.settings.mainBranch or task-level override
|
||||
...(baseBranch && { baseBranch })
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
import { mkdir, writeFile, readFile, stat } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import type { Project } from '../../../shared/types';
|
||||
import type { Project, TaskMetadata } from '../../../shared/types';
|
||||
import type { GitLabAPIIssue, GitLabAPINoteBasic, GitLabConfig } from './types';
|
||||
import { labelMatchesWholeWord } from '../shared/label-utils';
|
||||
import { sanitizeText, sanitizeStringArray } from '../shared/sanitize';
|
||||
@@ -460,11 +460,12 @@ export async function createSpecForIssue(
|
||||
await writeFile(metadataPath, JSON.stringify(metadata, null, 2), 'utf-8');
|
||||
|
||||
// Create task_metadata.json (consistent with GitHub format for backend compatibility)
|
||||
const taskMetadata = {
|
||||
const taskMetadata: TaskMetadata = {
|
||||
sourceType: 'gitlab' as const,
|
||||
gitlabIssueIid: safeIssue.iid,
|
||||
gitlabUrl: safeIssue.web_url,
|
||||
category: determineCategoryFromLabels(safeIssue.labels || []),
|
||||
userDescription: safeIssue.description || '',
|
||||
// Store baseBranch for worktree creation and QA comparison
|
||||
...(baseBranch && { baseBranch })
|
||||
};
|
||||
|
||||
@@ -244,6 +244,7 @@ export async function convertIdeaToTask(
|
||||
// Build task description and metadata
|
||||
const taskDescription = buildTaskDescription(idea);
|
||||
const metadata = buildTaskMetadata(idea);
|
||||
metadata.userDescription = taskDescription;
|
||||
|
||||
// Create spec files (inside lock to ensure atomicity)
|
||||
createSpecFiles(specDir, idea, taskDescription);
|
||||
|
||||
@@ -524,7 +524,8 @@ ${safeDescription || 'No description provided.'}
|
||||
linearIssueId: sanitizeText(issue.id, 100),
|
||||
linearIdentifier: safeIdentifier,
|
||||
linearUrl: safeUrl,
|
||||
category: 'feature'
|
||||
category: 'feature',
|
||||
userDescription: description,
|
||||
};
|
||||
// lgtm[js/http-to-file-access] - specDir is controlled, Linear data sanitized
|
||||
writeFileSync(path.join(specDir, 'task_metadata.json'), JSON.stringify(metadata, null, 2), 'utf-8');
|
||||
|
||||
@@ -585,6 +585,7 @@ ${(feature.acceptance_criteria || []).map((c: string) => `- [ ] ${c}`).join("\n"
|
||||
sourceType: "roadmap",
|
||||
featureId: feature.id,
|
||||
category: "feature",
|
||||
userDescription: taskDescription,
|
||||
};
|
||||
await writeFileWithRetry(path.join(specDir, "task_metadata.json"), JSON.stringify(metadata, null, 2), { encoding: 'utf-8' });
|
||||
|
||||
|
||||
@@ -1039,6 +1039,16 @@ export function registerSettingsHandlers(
|
||||
// Non-fatal: usage-monitor may use stale order until next app restart
|
||||
}
|
||||
|
||||
// Trigger immediate usage check so the new active profile's data is fetched
|
||||
// Without this, the UI shows "Usage data unavailable" until the next polling cycle
|
||||
try {
|
||||
const { getUsageMonitor } = await import('../claude-profile/usage-monitor');
|
||||
const monitor = getUsageMonitor();
|
||||
monitor.checkNow();
|
||||
} catch {
|
||||
// Non-fatal: usage will be fetched on next poll
|
||||
}
|
||||
|
||||
console.warn('[PROVIDER_ACCOUNTS_SET_QUEUE_ORDER] Queue order updated:', order.length, 'accounts');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
|
||||
@@ -207,10 +207,11 @@ export function registerTaskCRUDHandlers(agentManager: AgentManager): void {
|
||||
const specDir = path.join(specsDir, specId);
|
||||
mkdirSync(specDir, { recursive: true });
|
||||
|
||||
// Build metadata with source type
|
||||
// Build metadata with source type and preserve original user description
|
||||
const taskMetadata: TaskMetadata = {
|
||||
sourceType: 'manual',
|
||||
...metadata
|
||||
...metadata,
|
||||
userDescription: description,
|
||||
};
|
||||
|
||||
// Process and save attached images
|
||||
@@ -592,6 +593,11 @@ export function registerTaskCRUDHandlers(agentManager: AgentManager): void {
|
||||
updatedMetadata.attachedImages = savedImages;
|
||||
}
|
||||
|
||||
// Keep userDescription in sync when description is edited
|
||||
if (updates.description !== undefined) {
|
||||
updatedMetadata.userDescription = updates.description;
|
||||
}
|
||||
|
||||
// Sanitize thinking levels and update task_metadata.json
|
||||
sanitizeThinkingLevels(updatedMetadata);
|
||||
const metadataPath = path.join(specDir, 'task_metadata.json');
|
||||
|
||||
@@ -447,28 +447,48 @@ export class ProjectStore {
|
||||
}
|
||||
}
|
||||
|
||||
let description = '';
|
||||
const requirementsPath = path.join(specPath, AUTO_BUILD_PATHS.REQUIREMENTS);
|
||||
// PRIORITY 1: Read original user task description from requirements.json
|
||||
if (existsSync(requirementsPath)) {
|
||||
// Try to read task metadata (read early — userDescription is the preferred source)
|
||||
const metadataPath = path.join(specPath, 'task_metadata.json');
|
||||
let metadata: TaskMetadata | undefined;
|
||||
if (existsSync(metadataPath)) {
|
||||
try {
|
||||
const reqContent = readFileSync(requirementsPath, 'utf-8');
|
||||
const requirements = JSON.parse(reqContent);
|
||||
if (typeof requirements.task_description === 'string' && requirements.task_description.trim()) {
|
||||
// Use the full task description that the user entered
|
||||
description = requirements.task_description.trim();
|
||||
}
|
||||
const content = readFileSync(metadataPath, 'utf-8');
|
||||
metadata = JSON.parse(content);
|
||||
} catch {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
|
||||
// PRIORITY 2: Fallback to plan description if user requirement text is missing
|
||||
// Resolve task description with priority fallback:
|
||||
// PRIORITY 0: task_metadata.json userDescription (immutable — spec pipeline can't overwrite)
|
||||
// PRIORITY 1: requirements.json task_description (original format, before spec pipeline rewrites it)
|
||||
// PRIORITY 2: implementation_plan.json description (AI-generated summary)
|
||||
// PRIORITY 3: spec.md Overview section (AI-synthesized content)
|
||||
let description = '';
|
||||
|
||||
if (typeof metadata?.userDescription === 'string' && metadata.userDescription.trim()) {
|
||||
description = metadata.userDescription.trim();
|
||||
}
|
||||
|
||||
if (!description) {
|
||||
const requirementsPath = path.join(specPath, AUTO_BUILD_PATHS.REQUIREMENTS);
|
||||
if (existsSync(requirementsPath)) {
|
||||
try {
|
||||
const reqContent = readFileSync(requirementsPath, 'utf-8');
|
||||
const requirements = JSON.parse(reqContent);
|
||||
if (typeof requirements.task_description === 'string' && requirements.task_description.trim()) {
|
||||
description = requirements.task_description.trim();
|
||||
}
|
||||
} catch {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!description && plan?.description) {
|
||||
description = plan.description;
|
||||
}
|
||||
|
||||
// PRIORITY 3: Final fallback to spec.md Overview (AI-synthesized content)
|
||||
if (!description && existsSync(specFilePath)) {
|
||||
try {
|
||||
const content = readFileSync(specFilePath, 'utf-8');
|
||||
@@ -484,18 +504,6 @@ export class ProjectStore {
|
||||
}
|
||||
}
|
||||
|
||||
// Try to read task metadata
|
||||
const metadataPath = path.join(specPath, 'task_metadata.json');
|
||||
let metadata: TaskMetadata | undefined;
|
||||
if (existsSync(metadataPath)) {
|
||||
try {
|
||||
const content = readFileSync(metadataPath, 'utf-8');
|
||||
metadata = JSON.parse(content);
|
||||
} catch {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
|
||||
// Determine task status and review reason from plan
|
||||
// For JSON errors, store just the raw error - renderer will use i18n to format
|
||||
const finalDescription = hasJsonError
|
||||
|
||||
@@ -9,6 +9,7 @@ import { useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { RefreshCw, X } from 'lucide-react';
|
||||
import { Button } from './ui/button';
|
||||
import { loadSettings } from '../stores/settings-store';
|
||||
|
||||
interface SwapNotification {
|
||||
fromProfile: string;
|
||||
@@ -34,6 +35,15 @@ export function ProactiveSwapListener() {
|
||||
setNotification(notif);
|
||||
setIsVisible(true);
|
||||
|
||||
// Reload settings so the UI reflects the new active account
|
||||
// (proactive swap updates globalPriorityOrder in the settings file)
|
||||
loadSettings();
|
||||
|
||||
// Request fresh usage data for the new active profile
|
||||
// Without this, UsageIndicator shows "Usage data unavailable" until next poll
|
||||
window.electronAPI.requestUsageUpdate();
|
||||
window.electronAPI.requestAllProfilesUsage?.();
|
||||
|
||||
// Auto-hide after 5 seconds
|
||||
setTimeout(() => {
|
||||
setIsVisible(false);
|
||||
|
||||
@@ -440,14 +440,16 @@ export function UsageIndicator() {
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// No cached data for target — clear stale usage so it shows loading
|
||||
setUsage(null);
|
||||
// No cached data for target — keep old usage while waiting for fresh data
|
||||
// (setting null causes "Usage data unavailable" flash; the backend checkNow()
|
||||
// triggered by setQueueOrder will push fresh data via onUsageUpdated event)
|
||||
}
|
||||
|
||||
await setQueueOrder(newOrder);
|
||||
|
||||
// Fetch fresh data from backend
|
||||
window.electronAPI.requestUsageUpdate();
|
||||
// The queue order handler triggers checkNow() on the UsageMonitor,
|
||||
// which will emit 'usage-updated' with fresh data for the new active profile.
|
||||
// Also request all profiles to refresh the sidebar.
|
||||
window.electronAPI.requestAllProfilesUsage?.();
|
||||
}, [settings.globalPriorityOrder, providerAccounts, setQueueOrder, otherProfiles, usage, isCrossProviderMode]);
|
||||
|
||||
@@ -757,6 +759,7 @@ export function UsageIndicator() {
|
||||
const unsubscribeAllProfiles = window.electronAPI.onAllProfilesUsageUpdated?.((allProfilesUsage) => {
|
||||
// Filter out the active profile - we only want to show "other" profiles
|
||||
const nonActiveProfiles = allProfilesUsage.allProfiles.filter(p => !p.isActive);
|
||||
console.log('[UsageIndicator] Event: all profiles updated:', nonActiveProfiles.map(p => ({ id: p.profileId, name: p.profileName, session: p.sessionPercent, weekly: p.weeklyPercent })));
|
||||
setOtherProfiles(nonActiveProfiles);
|
||||
// Track if active profile needs re-auth
|
||||
const activeProfile = allProfilesUsage.allProfiles.find(p => p.isActive);
|
||||
@@ -781,6 +784,8 @@ export function UsageIndicator() {
|
||||
window.electronAPI.requestAllProfilesUsage?.().then((result) => {
|
||||
if (result.success && result.data) {
|
||||
const nonActiveProfiles = result.data.allProfiles.filter(p => !p.isActive);
|
||||
console.log('[UsageIndicator] Initial allProfiles received:', result.data.allProfiles.map(p => ({ id: p.profileId, name: p.profileName, active: p.isActive, session: p.sessionPercent, weekly: p.weeklyPercent })));
|
||||
console.log('[UsageIndicator] Non-active profiles for matching:', nonActiveProfiles.map(p => ({ id: p.profileId, name: p.profileName })));
|
||||
setOtherProfiles(nonActiveProfiles);
|
||||
// Track if active profile needs re-auth (even if main usage is unavailable)
|
||||
const activeProfile = result.data.allProfiles.find(p => p.isActive);
|
||||
@@ -865,6 +870,17 @@ export function UsageIndicator() {
|
||||
? otherProfiles.find(p => p.profileName === account.name || p.profileEmail === account.name)
|
||||
: undefined);
|
||||
|
||||
if (!profileData && hasOAuthMonitoring) {
|
||||
console.log('[UsageIndicator] No profileData match for account:', {
|
||||
accountId: account.id,
|
||||
accountName: account.name,
|
||||
claudeProfileId: account.claudeProfileId,
|
||||
provider: account.provider,
|
||||
authType: account.authType,
|
||||
otherProfileIds: otherProfiles.map(p => ({ id: p.profileId, name: p.profileName }))
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
key={account.id}
|
||||
|
||||
@@ -246,6 +246,9 @@ export interface TaskMetadata {
|
||||
useLocalBranch?: boolean; // If true, use the local branch directly instead of preferring origin/branch (preserves gitignored files)
|
||||
pushNewBranches?: boolean; // If false, keep the task branch local-only instead of auto-pushing to origin
|
||||
|
||||
// Original user description (preserved here because spec pipeline overwrites requirements.json)
|
||||
userDescription?: string;
|
||||
|
||||
// Archive status
|
||||
archivedAt?: string; // ISO date when task was archived
|
||||
archivedInVersion?: string; // Version in which task was archived (from changelog)
|
||||
|
||||
Reference in New Issue
Block a user