repair: let claude hand in its answer (StructuredOutput) through the hook
The second repair drill on VM103: sonnet found and fixed the broken health check through hl (write, up, ps until healthy), but its answer never arrived, because claude returns a --json-schema answer through its StructuredOutput tool and the PreToolUse hook refused every tool but Bash/Read/Edit/Write. StructuredOutput has no side effect: allowed in the hook and in --allowedTools of both profiles. A missing answer now names the refused tools. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
58311407f9
commit
0f2b21e32d
4 files changed
+21
-5
No files matched your search
@@ -30,13 +30,13 @@ const RESULT = (extra: object) => JSON.stringify({ type: 'result', subtype: 'suc
|
||||
describe('claudeArgs', () => {
|
||||
it('readonly: no tools at all', () => {
|
||||
expect(claudeArgs({ model: 'sonnet', profile: 'readonly', outputSchema: SCHEMA }, 'guard')).toEqual([
|
||||
'-p', '--output-format', 'json', '--model', 'sonnet', '--no-session-persistence', '--json-schema', JSON.stringify(SCHEMA), '--tools', '',
|
||||
'-p', '--output-format', 'json', '--model', 'sonnet', '--no-session-persistence', '--json-schema', JSON.stringify(SCHEMA), '--tools', '', '--allowedTools', 'StructuredOutput',
|
||||
]);
|
||||
});
|
||||
|
||||
it('tools: Bash only for hl, files, dontAsk, and the guard hook on every tool', () => {
|
||||
const args = claudeArgs({ model: 'opus', profile: 'tools', outputSchema: SCHEMA }, 'node guard.ts');
|
||||
expect(args.slice(args.indexOf('--tools'), args.indexOf('--settings'))).toEqual(['--tools', 'Bash,Read,Edit,Write', '--allowedTools', 'Bash(hl *)', 'Read', 'Edit', 'Write', '--permission-mode', 'dontAsk']);
|
||||
expect(args.slice(args.indexOf('--tools'), args.indexOf('--settings'))).toEqual(['--tools', 'Bash,Read,Edit,Write', '--allowedTools', 'Bash(hl *)', 'Read', 'Edit', 'Write', 'StructuredOutput', '--permission-mode', 'dontAsk']);
|
||||
expect(JSON.parse(args.at(-1)!)).toEqual({ hooks: { PreToolUse: [{ matcher: '.*', hooks: [{ type: 'command', command: 'node guard.ts' }] }] } });
|
||||
expect(args).not.toContain('--dangerously-skip-permissions');
|
||||
});
|
||||
@@ -59,7 +59,7 @@ describe('runClaude', () => {
|
||||
it.each([
|
||||
['a usage limit', RESULT({ is_error: true, subtype: 'error_during_execution', result: 'Claude AI usage limit reached' }), 'rate_limited'],
|
||||
['an expired login', RESULT({ is_error: true, subtype: 'error_during_execution', result: 'Invalid API key · Please run /login' }), 'auth_failed'],
|
||||
['no structured answer', RESULT({}), 'invalid_output'],
|
||||
['no structured answer', RESULT({ permission_denials: [{ tool_name: 'StructuredOutput', tool_use_id: 'x', tool_input: {} }] }), 'invalid_output'],
|
||||
['not JSON at all', 'Error: something broke', 'unavailable'],
|
||||
])('reports %s', async (_label, stdout, kind) => {
|
||||
const { deps } = setup(stdout, '', 1);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { randomBytes } from 'node:crypto';
|
||||
import { mkdir, rm } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { z } from 'zod';
|
||||
import { STRUCTURED_OUTPUT } from '../../domain/agent-guard.ts';
|
||||
import { describeError, truncate } from '../../domain/mask.ts';
|
||||
import type { CommandRunner, ModelFailure } from '../../domain/ports.ts';
|
||||
|
||||
@@ -49,6 +50,7 @@ const ClaudeOutput = z.object({
|
||||
structured_output: z.unknown().optional(),
|
||||
total_cost_usd: z.number().optional(),
|
||||
num_turns: z.number().optional(),
|
||||
permission_denials: z.array(z.object({ tool_name: z.string() })).optional(),
|
||||
});
|
||||
|
||||
const RATE_LIMITED = /rate.?limit|usage limit|limit (?:reached|exceeded)|\b429\b|overloaded/i;
|
||||
@@ -62,7 +64,8 @@ function classify(text: string): ModelFailure {
|
||||
|
||||
export function claudeArgs(request: Pick<ClaudeRunRequest, 'model' | 'profile' | 'outputSchema'>, guardCommand: string): string[] {
|
||||
const common = ['-p', '--output-format', 'json', '--model', request.model, '--no-session-persistence', '--json-schema', JSON.stringify(request.outputSchema)];
|
||||
if (request.profile === 'readonly') return [...common, '--tools', ''];
|
||||
// StructuredOutput is how claude returns the --json-schema answer: always allowed, in both profiles.
|
||||
if (request.profile === 'readonly') return [...common, '--tools', '', '--allowedTools', STRUCTURED_OUTPUT];
|
||||
const hooks = { hooks: { PreToolUse: [{ matcher: '.*', hooks: [{ type: 'command', command: guardCommand }] }] } };
|
||||
return [
|
||||
...common,
|
||||
@@ -73,6 +76,7 @@ export function claudeArgs(request: Pick<ClaudeRunRequest, 'model' | 'profile' |
|
||||
'Read',
|
||||
'Edit',
|
||||
'Write',
|
||||
STRUCTURED_OUTPUT,
|
||||
'--permission-mode',
|
||||
'dontAsk',
|
||||
'--settings',
|
||||
@@ -114,7 +118,10 @@ export async function runClaude(deps: ClaudeDeps, request: ClaudeRunRequest, sig
|
||||
if (!parsed.success) return { ok: false, kind: 'invalid_output', detail: `claude printed an unexpected result: ${parsed.error.issues[0]?.message ?? 'invalid'}` };
|
||||
const answer = parsed.data;
|
||||
if (answer.is_error) return { ok: false, kind: classify(answer.result ?? answer.subtype), detail: truncate(`claude: ${answer.subtype}: ${answer.result ?? ''}`, 500) };
|
||||
if (answer.structured_output === undefined) return { ok: false, kind: 'invalid_output', detail: 'claude gave no structured answer' };
|
||||
if (answer.structured_output === undefined) {
|
||||
const denied = parsed.data.permission_denials?.map((denial) => denial.tool_name).join(', ');
|
||||
return { ok: false, kind: 'invalid_output', detail: `claude gave no structured answer${denied === undefined || denied === '' ? '' : ` (refused tools: ${denied})`}: ${truncate(answer.result ?? '', 300)}` };
|
||||
}
|
||||
return { ok: true, output: answer.structured_output, costUsd: answer.total_cost_usd ?? 0, turns: answer.num_turns ?? 0 };
|
||||
} catch (cause) {
|
||||
if (signal?.aborted === true) throw cause;
|
||||
|
||||
@@ -147,6 +147,10 @@ describe('checkToolUse: the PreToolUse hook of the repairing model', () => {
|
||||
expect(checkToolUse('Read', {}, cwd).ok).toBe(false);
|
||||
});
|
||||
|
||||
it('lets the model hand in its answer (StructuredOutput, how claude answers --json-schema)', () => {
|
||||
expect(checkToolUse('StructuredOutput', { outcome: 'fixed', summary: 'x', actions: [] }, cwd)).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it('refuses every other tool', () => {
|
||||
for (const tool of ['WebFetch', 'WebSearch', 'Task', 'NotebookEdit', 'mcp__x__y']) expect(checkToolUse(tool, {}, cwd).ok).toBe(false);
|
||||
});
|
||||
|
||||
@@ -71,6 +71,9 @@ function normalizePath(path: string): string {
|
||||
}
|
||||
const within = (path: string, root: string): boolean => path === root || path.startsWith(root === '/' ? '/' : `${root}/`);
|
||||
|
||||
/** The tool claude answers a `--json-schema` request with. */
|
||||
export const STRUCTURED_OUTPUT = 'StructuredOutput';
|
||||
|
||||
/** Whether `path` (absolute, or relative to `dir`) stays inside `dir` once `..` and `.` are resolved. */
|
||||
export function isInside(path: string, dir: string): boolean {
|
||||
const root = normalizePath(dir);
|
||||
@@ -79,6 +82,8 @@ export function isInside(path: string, dir: string): boolean {
|
||||
|
||||
/** The PreToolUse decision for the repairing model: Bash runs `hl` only; files only inside the session's directory. */
|
||||
export function checkToolUse(tool: string, input: Readonly<Record<string, unknown>>, cwd: string): { ok: true } | { ok: false; reason: string } {
|
||||
// How claude returns the --json-schema answer: no side effect. Refusing it left the first repair drill without an answer.
|
||||
if (tool === STRUCTURED_OUTPUT) return { ok: true };
|
||||
if (tool === 'Bash') return checkHlCommand(typeof input['command'] === 'string' ? input['command'] : '');
|
||||
if (tool === 'Read' || tool === 'Edit' || tool === 'Write' || tool === 'MultiEdit') {
|
||||
const path = input['file_path'];
|
||||
|
||||
Reference in new issue
Block a user