fix(registry): wait out a ghcr.io rate limit instead of reporting the version unknown

ghcr.io answered 429 to the tag list for jackett, radarr and sonarr, and image_tags
turned that straight into version_unknown with only "skip" left. The registry client
now asks again after 429, honouring Retry-After up to five minutes; the driver waits
30 s, 2 min and 5 min, the server does not wait.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/infra/http-registry.test.ts

@@ -137,6 +137,47 @@ describe('createHttpRegistryClient', () => {
     });
   });
 
+  describe('HTTP 429', () => {
+    const tooMany = (headers: Record<string, string> = {}) => new Response('', { status: 429, headers });
+
+    it('waits and asks again, so a short rate limit does not fail the lookup', async () => {
+      const waits: number[] = [];
+      const { fetchFn, calls } = fakeRegistry({ [HUB]: [tooMany(), tooMany(), json({ tags: ['1'] })] });
+      const client = createHttpRegistryClient({ fetch: fetchFn, rateLimitWaitsMs: [5_000, 30_000], sleep: (ms) => (waits.push(ms), Promise.resolve()) });
+      expect(await client.listTags('nextcloud')).toEqual(['1']);
+      expect(waits).toEqual([5_000, 30_000]);
+      expect(calls).toHaveLength(3);
+    });
+
+    it('waits as long as Retry-After asks when that is longer, up to five minutes', async () => {
+      const waits: number[] = [];
+      const { fetchFn } = fakeRegistry({ [HUB]: [tooMany({ 'retry-after': '20' }), tooMany({ 'retry-after': '3600' }), json({ tags: [] })] });
+      await createHttpRegistryClient({ fetch: fetchFn, rateLimitWaitsMs: [5_000, 30_000], sleep: (ms) => (waits.push(ms), Promise.resolve()) }).listTags('nextcloud');
+      expect(waits).toEqual([20_000, 300_000]);
+    });
+
+    it('does not ask again unless pauses are given', async () => {
+      const { fetchFn, calls } = fakeRegistry({ [HUB]: tooMany() });
+      await createHttpRegistryClient({ fetch: fetchFn }).listTags('nextcloud').catch(() => undefined);
+      expect(calls).toHaveLength(1);
+    });
+
+    it('gives up as rate_limited once the pauses are used', async () => {
+      const { fetchFn, calls } = fakeRegistry({ [HUB]: tooMany() });
+      const error = await createHttpRegistryClient({ fetch: fetchFn, rateLimitWaitsMs: [1] }).listTags('nextcloud').catch((cause: unknown) => cause);
+      expect((error as RegistryError).kind).toBe('rate_limited');
+      expect(calls).toHaveLength(2);
+    });
+
+    it('an abort during the pause is rethrown as it is', async () => {
+      const controller = new AbortController();
+      const { fetchFn } = fakeRegistry({ [HUB]: tooMany() });
+      const running = createHttpRegistryClient({ fetch: fetchFn, rateLimitWaitsMs: [5_000] }).listTags('nextcloud', controller.signal);
+      setTimeout(() => controller.abort(new Error('step time limit')), 20);
+      await expect(running).rejects.toThrow('step time limit');
+    });
+  });
+
   describe('time', () => {
     const hanging = (_input: string | URL | Request, init?: RequestInit) =>
       new Promise<Response>((_resolve, reject) => init?.signal?.addEventListener('abort', () => reject(init.signal?.reason as Error)));

src/infra/http-registry.ts

@@ -1,4 +1,5 @@
 import { createHash } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
 import { z } from 'zod';
 import { findPlatformEntry, parseImageRef, platformNames } from '../domain/image-ref.ts';
 import type { IndexEntry } from '../domain/image-ref.ts';
@@ -14,11 +15,16 @@ export interface HttpRegistryOptions {
   readonly maxPages?: number;
   /** Largest response body we read (tag pages, manifests, image configs). */
   readonly maxBodyBytes?: number;
+  /** Pauses before asking again after HTTP 429, one retry per entry; none by default. ghcr.io answers 429 now and then and is fine later. */
+  readonly rateLimitWaitsMs?: readonly number[];
+  readonly sleep?: (ms: number, signal: AbortSignal | undefined) => Promise<void>;
 }
 
 const DEFAULT_TIMEOUT_MS = 30_000;
 const DEFAULT_MAX_PAGES = 50;
 const DEFAULT_MAX_BODY_BYTES = 4 * 1024 * 1024;
+/** A longer `Retry-After` than this is not waited out. */
+const MAX_RATE_LIMIT_WAIT_MS = 300_000;
 
 const MANIFEST_ACCEPT = [
   'application/vnd.oci.image.index.v1+json',
@@ -58,6 +64,8 @@ export function createHttpRegistryClient(options: HttpRegistryOptions = {}): Reg
   const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
   const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES;
+  const rateLimitWaitsMs = options.rateLimitWaitsMs ?? [];
+  const sleep = options.sleep ?? ((ms: number, signal: AbortSignal | undefined) => delay(ms, undefined, signal === undefined ? {} : { signal }));
 
   /** One repository on one registry: the token is fetched once and reused by every request. */
   function openSession(location: RepoLocation, signal: AbortSignal | undefined) {
@@ -65,8 +73,25 @@ export function createHttpRegistryClient(options: HttpRegistryOptions = {}): Reg
     const what = `${location.registry}/${location.path}`;
     let token: string | null = null;
 
-    /** One request with the caller's abort and our own time limit. A caller abort is rethrown untouched. */
+    /** `request`, asked again after HTTP 429 as long as `rateLimitWaitsMs` has pauses left; the last 429 is returned. */
     async function request(url: string, authorization: string | null, accept?: string): Promise<Response> {
+      for (let attempt = 0; ; attempt++) {
+        const response = await requestOnce(url, authorization, accept);
+        const wait = rateLimitWaitsMs[attempt];
+        if (response.status !== 429 || wait === undefined) return response;
+        const retryAfterMs = Number(response.headers.get('retry-after')) * 1000;
+        const pause = Number.isFinite(retryAfterMs) && retryAfterMs > wait ? Math.min(retryAfterMs, MAX_RATE_LIMIT_WAIT_MS) : wait;
+        void response.body?.cancel().catch(() => undefined);
+        try {
+          await sleep(pause, signal);
+        } catch (cause) {
+          throw signal?.aborted === true ? (signal.reason as Error) : cause;
+        }
+      }
+    }
+
+    /** One request with the caller's abort and our own time limit. A caller abort is rethrown untouched. */
+    async function requestOnce(url: string, authorization: string | null, accept?: string): Promise<Response> {
       const timeout = AbortSignal.timeout(timeoutMs);
       const combined = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
       const headers: Record<string, string> = {};

src/main/driver.ts

@@ -42,6 +42,8 @@ import { missingSkills, readSkillFiles } from './skills-file.ts';
 import { createWorkflowExtensions } from '../domain/workflow-extensions.ts';
 
 const logger = createConsoleLogger();
+/** ghcr.io answers 429 now and then; the schedule is spread out, so a run can wait minutes for the tag list. */
+const REGISTRY_RATE_LIMIT_WAITS_MS = [30_000, 120_000, 300_000];
 
 let config;
 try {
@@ -118,7 +120,7 @@ const runner = createWorkflowRunner({
   functionCatalog,
   functionRuntime: createFunctionRuntimeClient({ baseUrl: config.functionRuntimeUrl }),
   componentOperations: createComponentOperationExecutor({
-    registry: createHttpRegistryClient(),
+    registry: createHttpRegistryClient({ rateLimitWaitsMs: REGISTRY_RATE_LIMIT_WAITS_MS }),
     hostBackup,
     backupVmid: config.backupVmid,
     clock: systemClock,