413 lines
15 KiB
TypeScript
413 lines
15 KiB
TypeScript
import { describe, expect, it, vi } from 'vitest';
|
|
import { existsSync, mkdirSync, readFileSync, writeFileSync, mkdtempSync } from 'fs';
|
|
import { join } from 'path';
|
|
import { tmpdir } from 'os';
|
|
import { buildRetryHandoffSummary, Worker } from './worker.js';
|
|
import { userPiecesDir } from './user-folder/paths.js';
|
|
import { loadPiece } from './engine/piece-runner.js';
|
|
import type { AppConfig } from './config.js';
|
|
import type { Job } from './db/repository.js';
|
|
|
|
function makeConfig(): AppConfig {
|
|
return {
|
|
provider: {
|
|
model: 'test-model',
|
|
workers: [{ id: 'worker-1', endpoint: 'http://localhost:11434/v1' }],
|
|
},
|
|
worktreeDir: '/tmp/worker-test',
|
|
concurrency: 1,
|
|
maxMovements: 30,
|
|
retry: { maxAttempts: 3, backoffSeconds: [60, 300, 900] },
|
|
ask: { maxPerJob: 2 },
|
|
subtasks: { maxDepth: 2 },
|
|
tools: {
|
|
searxngUrl: 'http://localhost:8080',
|
|
visionModel: 'vision',
|
|
visionTimeout: 60,
|
|
visionMaxTokens: 1024,
|
|
webfetchTimeout: 30,
|
|
websearchTimeout: 15,
|
|
webfetchAllowedHosts: [],
|
|
},
|
|
};
|
|
}
|
|
|
|
function makeJob(): Job {
|
|
return {
|
|
id: 'job-1',
|
|
repo: 'acme/demo',
|
|
issueNumber: 12,
|
|
prNumber: null,
|
|
status: 'running',
|
|
pieceName: 'general',
|
|
requiredRole: 'auto',
|
|
requiredProfile: 'auto',
|
|
currentMovement: null,
|
|
instruction: 'fix it',
|
|
branchName: null,
|
|
worktreePath: null,
|
|
attempt: 1,
|
|
maxAttempts: 3,
|
|
nextRetryAt: null,
|
|
errorSummary: null,
|
|
resumeMovement: null,
|
|
askCount: 0,
|
|
workerId: 'worker-1',
|
|
parentJobId: null,
|
|
subtaskDepth: 0,
|
|
createdAt: new Date().toISOString(),
|
|
updatedAt: new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
describe('Worker', () => {
|
|
it('requeues a claimed job when the issue lock is already held', async () => {
|
|
const repo = {
|
|
lockIssue: vi.fn().mockResolvedValue(false),
|
|
updateJob: vi.fn().mockResolvedValue(undefined),
|
|
addAuditLog: vi.fn().mockResolvedValue(undefined),
|
|
};
|
|
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:11434/v1',
|
|
'test-model',
|
|
repo as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
await (worker as unknown as { executeJob: (job: Job) => Promise<void> }).executeJob(makeJob());
|
|
|
|
expect(repo.lockIssue).toHaveBeenCalledWith('acme/demo', 12, 'job-1');
|
|
expect(repo.updateJob).toHaveBeenCalledWith('job-1', { status: 'queued', workerId: null });
|
|
expect(repo.addAuditLog).toHaveBeenCalledWith('job-1', 'job_requeued_issue_locked', 'worker', {
|
|
workerId: 'worker-1',
|
|
});
|
|
});
|
|
|
|
it('waitForCompletion resolves immediately when not processing', async () => {
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:11434/v1',
|
|
'test-model',
|
|
{} as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
const result = await worker.waitForCompletion(1000);
|
|
expect(result).toBe(true);
|
|
});
|
|
|
|
it('exposes workerId via id getter', () => {
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:11434/v1',
|
|
'test-model',
|
|
{} as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
expect(worker.id).toBe('worker-1');
|
|
});
|
|
|
|
it('requeues jobs and marks the worker unhealthy on LLM connection errors', async () => {
|
|
const repo = {
|
|
updateWorkerNodeHealth: vi.fn().mockResolvedValue(undefined),
|
|
updateJob: vi.fn().mockResolvedValue(undefined),
|
|
};
|
|
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:11434/v1',
|
|
'test-model',
|
|
repo as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
await (worker as unknown as {
|
|
scheduleRetryOrFail: (job: Job, errorMsg: string) => Promise<string>;
|
|
}).scheduleRetryOrFail(makeJob(), 'LLM error: Connection error: fetch failed');
|
|
|
|
expect(repo.updateWorkerNodeHealth).toHaveBeenCalledWith('worker-1', {
|
|
healthy: false,
|
|
lastError: 'LLM error: Connection error: fetch failed',
|
|
inflightJobs: 0,
|
|
availableModels: [],
|
|
});
|
|
expect(repo.updateJob).toHaveBeenCalledWith('job-1', {
|
|
status: 'queued',
|
|
workerId: null,
|
|
errorSummary: 'LLM error: Connection error: fetch failed',
|
|
abortReason: null,
|
|
nextRetryAt: null,
|
|
});
|
|
});
|
|
|
|
it('writes retry handoff summary when scheduling a retry', async () => {
|
|
const workspacePath = mkdtempSync(join(tmpdir(), 'retry-handoff-'));
|
|
const repo = {
|
|
updateJob: vi.fn().mockResolvedValue(undefined),
|
|
};
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:11434/v1',
|
|
'test-model',
|
|
repo as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
await (worker as unknown as {
|
|
scheduleRetryOrFail: (job: Job, errorMsg: string, workspacePath?: string) => Promise<string>;
|
|
}).scheduleRetryOrFail(makeJob(), 'LLM error 400: context overflow', workspacePath);
|
|
|
|
const summaryPath = join(workspacePath, 'logs', 'retry-summary.md');
|
|
expect(existsSync(summaryPath)).toBe(true);
|
|
const summary = readFileSync(summaryPath, 'utf-8');
|
|
expect(summary).toContain('# Retry Handoff');
|
|
expect(summary).toContain('Disposition: retry');
|
|
expect(summary).toContain('LLM error 400: context overflow');
|
|
expect(summary).toContain('次のエージェントへの指示');
|
|
});
|
|
|
|
it('builds retry handoff summary from diagnostics and lessons', () => {
|
|
const workspacePath = mkdtempSync(join(tmpdir(), 'retry-handoff-context-'));
|
|
mkdirSync(join(workspacePath, 'logs'), { recursive: true });
|
|
writeFileSync(join(workspacePath, 'logs', 'last-run-diagnostics.json'), JSON.stringify({
|
|
status: 'error',
|
|
abortReason: 'movement_without_transition',
|
|
finalOutput: '前回は analyze で失敗した',
|
|
movementHistory: [
|
|
{ name: 'gather', next: 'analyze', toolsUsed: ['Read'], outputPreview: 'input を確認済み' },
|
|
{ name: 'analyze', next: 'ABORT', toolsUsed: ['Grep'], outputPreview: 'context overflow' },
|
|
],
|
|
contextActions: [{ type: 'prompt' }],
|
|
}), 'utf-8');
|
|
writeFileSync(join(workspacePath, 'logs', 'lessons.jsonl'), [
|
|
JSON.stringify({ movement: 'gather', lessons: '大きな HTML は Grep で絞る' }),
|
|
JSON.stringify({ movement: 'analyze', lessons: 'Read は offset/limit を使う' }),
|
|
].join('\n'), 'utf-8');
|
|
|
|
const summary = buildRetryHandoffSummary({
|
|
workspacePath,
|
|
job: makeJob(),
|
|
errorMsg: 'LLM error 400',
|
|
nextRetryAt: '2026-04-28T00:00:00.000Z',
|
|
disposition: 'retry',
|
|
});
|
|
|
|
expect(summary).toContain('movement gather -> analyze');
|
|
expect(summary).toContain('input を確認済み');
|
|
expect(summary).toContain('[gather] 大きな HTML は Grep で絞る');
|
|
expect(summary).toContain('Next retry at: 2026-04-28T00:00:00.000Z');
|
|
});
|
|
|
|
it('initialize() stays healthy when model is undefined and model-list endpoints are unavailable (llama-server compat)', async () => {
|
|
const fetchMock = vi.fn(async (input: RequestInfo | URL) => {
|
|
const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url;
|
|
if (url.includes('/api/tags') || url.endsWith('/models') || url.includes('/v1/models')) {
|
|
return { ok: false, status: 404, json: async () => ({}) } as Response;
|
|
}
|
|
if (url.includes('/props')) {
|
|
return { ok: true, status: 200, json: async () => ({ n_ctx: 8192 }) } as Response;
|
|
}
|
|
return { ok: false, status: 404, json: async () => ({}) } as Response;
|
|
});
|
|
const origFetch = globalThis.fetch;
|
|
globalThis.fetch = fetchMock as never;
|
|
|
|
const warnSpy = vi.fn();
|
|
const loggerMod = await import('./logger.js');
|
|
const origWarn = loggerMod.logger.warn;
|
|
loggerMod.logger.warn = warnSpy as never;
|
|
|
|
try {
|
|
const repo = {
|
|
upsertWorkerNode: vi.fn().mockResolvedValue(undefined),
|
|
};
|
|
// model 引数を undefined にして llama-server compat モードを再現
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:8080/v1',
|
|
undefined,
|
|
repo as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
const ok = await worker.initialize();
|
|
expect(ok).toBe(true);
|
|
|
|
const upsertCall = repo.upsertWorkerNode.mock.calls.at(-1)?.[0];
|
|
expect(upsertCall?.healthy).toBe(true);
|
|
expect(upsertCall?.availableModels).toEqual([]);
|
|
expect(upsertCall?.lastError).toBeNull();
|
|
|
|
// model 未設定では model-list 取得失敗を WARN しない
|
|
const failedFetchWarn = warnSpy.mock.calls.find(c => String(c[0]).includes('failed to fetch model list'));
|
|
expect(failedFetchWarn).toBeUndefined();
|
|
} finally {
|
|
globalThis.fetch = origFetch;
|
|
loggerMod.logger.warn = origWarn;
|
|
}
|
|
});
|
|
|
|
it('initialize() forwards Authorization: Bearer apiKey on /api/tags and /v1/models probes', async () => {
|
|
// Regression for the 2026-05-20 dogfooding finding: workers pointed
|
|
// at AAO Gateway (Bearer-auth-required /v1/models) saw 30s-interval
|
|
// 401 floods because the discovery probes were sent un-authenticated.
|
|
// After the fix, the same apiKey used for chat completions must be
|
|
// forwarded to the discovery probes too.
|
|
const seenAuth: string[] = [];
|
|
const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url;
|
|
const auth = (init?.headers as Record<string, string> | undefined)?.['Authorization'];
|
|
if (typeof auth === 'string') seenAuth.push(`${url} :: ${auth}`);
|
|
if (url.endsWith('/api/tags')) {
|
|
return { ok: false, status: 401, json: async () => ({}) } as Response;
|
|
}
|
|
if (url.endsWith('/models') || url.endsWith('/v1/models')) {
|
|
return {
|
|
ok: true, status: 200,
|
|
json: async () => ({ data: [{ id: 'dogfood-gpu-1' }] }),
|
|
} as Response;
|
|
}
|
|
return { ok: false, status: 404, json: async () => ({}) } as Response;
|
|
});
|
|
const origFetch = globalThis.fetch;
|
|
globalThis.fetch = fetchMock as never;
|
|
|
|
try {
|
|
const repo = { upsertWorkerNode: vi.fn().mockResolvedValue(undefined) };
|
|
const cfg = makeConfig();
|
|
cfg.provider.workers = [{
|
|
id: 'worker-1',
|
|
endpoint: 'http://localhost:4000/v1',
|
|
apiKey: 'sk-aao-test-key',
|
|
}];
|
|
const worker = new Worker(
|
|
'worker-1', 'http://localhost:4000/v1', 'dogfood-gpu-1',
|
|
repo as never, cfg,
|
|
);
|
|
|
|
const ok = await worker.initialize();
|
|
expect(ok).toBe(true);
|
|
|
|
// Authorization header must be present on BOTH probe attempts.
|
|
expect(seenAuth.length).toBeGreaterThanOrEqual(2);
|
|
expect(seenAuth.every(s => s.includes('Bearer sk-aao-test-key'))).toBe(true);
|
|
|
|
// Models discovered through the OpenAI fallback (with auth) get
|
|
// stored as availableModels.
|
|
const upsertCall = repo.upsertWorkerNode.mock.calls.at(-1)?.[0];
|
|
expect(upsertCall?.availableModels).toEqual(['dogfood-gpu-1']);
|
|
} finally {
|
|
globalThis.fetch = origFetch;
|
|
}
|
|
});
|
|
|
|
it('initialize() still WARNs when model is configured and endpoints are unavailable', async () => {
|
|
const fetchMock = vi.fn(async () => {
|
|
return { ok: false, status: 404, json: async () => ({}) } as Response;
|
|
});
|
|
const origFetch = globalThis.fetch;
|
|
globalThis.fetch = fetchMock as never;
|
|
|
|
const warnSpy = vi.fn();
|
|
const loggerMod = await import('./logger.js');
|
|
const origWarn = loggerMod.logger.warn;
|
|
loggerMod.logger.warn = warnSpy as never;
|
|
|
|
try {
|
|
const repo = {
|
|
upsertWorkerNode: vi.fn().mockResolvedValue(undefined),
|
|
};
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:11434/v1',
|
|
'qwen3:32b',
|
|
repo as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
const ok = await worker.initialize();
|
|
expect(ok).toBe(false);
|
|
|
|
const failedFetchWarn = warnSpy.mock.calls.find(c => String(c[0]).includes('failed to fetch model list'));
|
|
expect(failedFetchWarn).toBeDefined();
|
|
} finally {
|
|
globalThis.fetch = origFetch;
|
|
loggerMod.logger.warn = origWarn;
|
|
}
|
|
});
|
|
|
|
it('cancelCheck returns true when job status is cancelled', () => {
|
|
const repo = {
|
|
getJobStatusSync: vi.fn().mockReturnValue('cancelled'),
|
|
};
|
|
|
|
const worker = new Worker(
|
|
'worker-1',
|
|
'http://localhost:11434/v1',
|
|
'test-model',
|
|
repo as never,
|
|
makeConfig(),
|
|
);
|
|
|
|
// cancelCheck は executeJob 内で生成されるクロージャ。
|
|
// getJobStatusSync を直接呼んで正しい値を返すことを確認する。
|
|
const status = repo.getJobStatusSync('job-1');
|
|
expect(status).toBe('cancelled');
|
|
expect(repo.getJobStatusSync).toHaveBeenCalledWith('job-1');
|
|
void worker; // worker インスタンスを参照して lint 警告を回避
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Regression: null-ownerId job resolves pieces from data/users/local/pieces
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('Worker piece resolution for null-ownerId job', () => {
|
|
it('a no-auth job (ownerId null) can load a piece from data/users/local/pieces', () => {
|
|
const tempDir = mkdtempSync(join(tmpdir(), 'worker-piece-res-'));
|
|
const piecesDir = join(tempDir, 'pieces');
|
|
const userFolderRoot = join(tempDir, 'users');
|
|
mkdirSync(piecesDir);
|
|
|
|
// Write a piece under local user-custom dir (where no-auth POST now writes)
|
|
const localPiecesDir = userPiecesDir(userFolderRoot, 'local');
|
|
mkdirSync(localPiecesDir, { recursive: true });
|
|
writeFileSync(join(localPiecesDir, 'local-piece.yaml'), [
|
|
'name: local-piece',
|
|
'description: local custom piece',
|
|
'max_movements: 1',
|
|
'initial_movement: only',
|
|
'movements:',
|
|
' - name: only',
|
|
' edit: false',
|
|
' persona: p',
|
|
' instruction: i',
|
|
' allowed_tools: [Read]',
|
|
' default_next: COMPLETE',
|
|
' rules: []',
|
|
].join('\n'));
|
|
|
|
// Simulate what worker.ts does: ownerForPieces = job.ownerId ?? 'local'
|
|
const ownerForPieces = (null as string | null) ?? 'local';
|
|
const customPieceDirs = [userPiecesDir(userFolderRoot, ownerForPieces)].filter(Boolean);
|
|
|
|
// loadPiece should find the piece in the local user-custom dir
|
|
const piece = loadPiece('local-piece', piecesDir, customPieceDirs);
|
|
expect(piece.name).toBe('local-piece');
|
|
expect(piece.description).toBe('local custom piece');
|
|
});
|
|
|
|
it('a null-ownerId job resolves the local user-custom dir path consistently', () => {
|
|
const userFolderRoot = '/data/users';
|
|
// The fixed worker uses: job.ownerId ?? 'local'
|
|
const ownerForNullJob = (null as string | null) ?? 'local';
|
|
const resolvedDir = userPiecesDir(userFolderRoot, ownerForNullJob);
|
|
expect(resolvedDir).toContain('local');
|
|
// Same dir that no-auth POST writes to
|
|
expect(resolvedDir).toBe(userPiecesDir(userFolderRoot, 'local'));
|
|
});
|
|
});
|