maestro/src/engine/tools/review.test.ts

import * as fs from 'fs';
import * as path from 'path';
import { tmpdir } from 'os';
import { afterEach, describe, expect, it, vi } from 'vitest';
import type { Message } from '../../llm/openai-compat.js';
import type { ToolContext } from './core.js';
import { executeTool } from './review.js';

function makeWorkspace(): string {
  return fs.mkdtempSync(path.join(tmpdir(), 'maestro-review-'));
}

function makeContext(workspacePath: string, runner?: (messages: Message[]) => Promise<string>): ToolContext {
  return {
    workspacePath,
    editAllowed: true,
    runIsolatedLlm: runner,
  };
}

describe('review tools', () => {
  let workspacePath = '';

  afterEach(() => {
    if (workspacePath) {
      fs.rmSync(workspacePath, { recursive: true, force: true });
      workspacePath = '';
    }
    vi.restoreAllMocks();
  });

  it('reviews multiple text files with isolated LLM calls', async () => {
    workspacePath = makeWorkspace();
    fs.mkdirSync(path.join(workspacePath, 'output', 'ocr'), { recursive: true });
    fs.writeFileSync(path.join(workspacePath, 'output', 'ocr', 'a.md'), 'hostname: sw-01');
    fs.writeFileSync(path.join(workspacePath, 'output', 'ocr', 'b.md'), 'hostname: sw-02');

    const runner = vi.fn(async (messages: Message[]) => {
      const userMessage = messages.find((message) => message.role === 'user')?.content ?? '';
      const file = /Source file: ([^\n]+)/.exec(userMessage)?.[1] ?? 'unknown';
      return JSON.stringify({
        source_file: file,
        summary: `reviewed ${file}`,
        quality: 'good',
        needs_retry: false,
        extracted_items: { hostname: file.includes('a.md') ? 'sw-01' : 'sw-02' },
        missing_items: [],
        notes: [],
      });
    });

    const result = await executeTool('BatchReviewTextWithLLM', {
      input_glob: 'output/ocr/*.md',
      review_prompt: 'Extract config values and assess OCR quality',
    }, makeContext(workspacePath, runner));

    expect(result?.isError).toBe(false);
    expect(runner).toHaveBeenCalledTimes(2);
    expect(fs.existsSync(path.join(workspacePath, 'output', 'reviewed', 'output_ocr__a.json'))).toBe(true);
    expect(fs.existsSync(path.join(workspacePath, 'output', 'reviewed', 'output_ocr__b.json'))).toBe(true);
    const manifest = fs.readFileSync(path.join(workspacePath, 'output', 'reviewed', 'manifest.json'), 'utf-8');
    expect(manifest).toContain('sw-01');
    expect(manifest).toContain('sw-02');
  });

  it('merges reviewed JSON files into one markdown file', async () => {
    workspacePath = makeWorkspace();
    fs.mkdirSync(path.join(workspacePath, 'output', 'reviewed'), { recursive: true });
    fs.writeFileSync(path.join(workspacePath, 'output', 'reviewed', 'a.json'), JSON.stringify({
      source_file: 'output/ocr/a.md',
      summary: 'good OCR',
      quality: 'good',
      needs_retry: false,
      extracted_items: { hostname: 'sw-01' },
      missing_items: [],
      notes: [],
    }, null, 2));
    fs.writeFileSync(path.join(workspacePath, 'output', 'reviewed', 'b.json'), JSON.stringify({
      source_file: 'output/ocr/b.md',
      summary: 'needs retry',
      quality: 'partial',
      needs_retry: true,
      extracted_items: { hostname: 'sw-02' },
      missing_items: ['gateway'],
      notes: ['blurred right edge'],
    }, null, 2));

    const result = await executeTool('MergeReviewedResults', {
      input_glob: 'output/reviewed/*.json',
      output_path: 'output/reports/final-summary.md',
    }, makeContext(workspacePath));

    expect(result?.isError).toBe(false);
    const summary = fs.readFileSync(path.join(workspacePath, 'output', 'reports', 'final-summary.md'), 'utf-8');
    expect(summary).toContain('Reviewed Results Summary');
    expect(summary).toContain('output/ocr/a.md');
    expect(summary).toContain('needs retry');
    expect(summary).toContain('gateway');
  });

  it('rejects review outputs outside tool-specific prefixes', async () => {
    workspacePath = makeWorkspace();
    fs.mkdirSync(path.join(workspacePath, 'output', 'ocr'), { recursive: true });
    fs.writeFileSync(path.join(workspacePath, 'output', 'ocr', 'a.md'), 'hostname: sw-01');

    const runner = vi.fn(async () => JSON.stringify({
      source_file: 'output/ocr/a.md',
      summary: 'ok',
      quality: 'good',
      needs_retry: false,
      extracted_items: {},
      missing_items: [],
      notes: [],
    }));

    const blockedBatch = await executeTool('BatchReviewTextWithLLM', {
      input_glob: 'output/ocr/*.md',
      review_prompt: 'review',
      output_dir: 'output/misc',
    }, makeContext(workspacePath, runner));

    expect(blockedBatch?.isError).toBe(true);
    expect(blockedBatch?.output).toContain('output/reviewed');

    fs.mkdirSync(path.join(workspacePath, 'output', 'reviewed'), { recursive: true });
    fs.writeFileSync(path.join(workspacePath, 'output', 'reviewed', 'a.json'), JSON.stringify({
      source_file: 'output/ocr/a.md',
      summary: 'ok',
      quality: 'good',
      needs_retry: false,
      extracted_items: {},
      missing_items: [],
      notes: [],
    }, null, 2));

    const blockedMerge = await executeTool('MergeReviewedResults', {
      input_glob: 'output/reviewed/*.json',
      output_path: 'output/final-summary.md',
    }, makeContext(workspacePath));

    expect(blockedMerge?.isError).toBe(true);
    expect(blockedMerge?.output).toContain('output/reports');
  });
});