Skip to content

fix(annotator): use shared fetchWithRetry in CourtListener client #275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
williamzujkowski merged 1 commit into from
Oct 6, 2026
Merged

fix(annotator): use shared fetchWithRetry in CourtListener client #275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions packages/annotator/src/__tests__/client.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,21 @@ describe('CourtListenerClient.searchByStatute (#237)', () => {
if (!result.ok) return;
expect(result.value).toEqual([]);
});

it('aborts immediately when signal is cancelled without retrying (#223 item 2)', async () => {
const controller = new AbortController();
controller.abort();

const client = new CourtListenerClient({ token: 't', logger: createLogger('test', 'error') });
const result = await client.searchByStatute('18 U.S.C. 111', { signal: controller.signal });

expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.error.name).toBe('AbortError');
});
});


describe('readJsonCapped — response size cap (#223 item 3)', () => {
const CAP = MAX_API_RESPONSE_BYTES;

Expand Down
80 changes: 24 additions & 56 deletions packages/annotator/src/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { type Result, ok, err } from '@civic-source/types';
import { type Logger, MAX_RETRIES, BASE_BACKOFF_MS, TokenBucket } from '@civic-source/shared';
import { type Logger, TokenBucket, fetchWithRetry } from '@civic-source/shared';
import {
COURTLISTENER_BASE_URL,
SEARCH_ENDPOINT,
Expand Down Expand Up @@ -133,10 +133,6 @@ function hasResultsArray(data: unknown): data is { results: unknown[] } {
return Array.isArray(obj['results']);
}

function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

/**
* CourtListener API client with retry logic and token bucket rate limiting.
*
Expand All @@ -160,7 +156,10 @@ export class CourtListenerClient {
* Search for opinions mentioning a statute section.
* Uses full-text search since statute citations are not structured fields.
*/
async searchByStatute(section: string): Promise<Result<CourtListenerResult[]>> {
async searchByStatute(
section: string,
options?: { signal?: AbortSignal }
): Promise<Result<CourtListenerResult[]>> {
if (!this.rateLimiter.tryConsume()) {
this.logger.warn('Rate limited, waiting for token', { section });
await this.rateLimiter.waitAndConsume();
Expand All @@ -174,10 +173,25 @@ export class CourtListenerClient {

this.logger.info('Searching CourtListener', { section, url: url.toString() });

const result = await this.fetchWithRetry(url.toString());
if (!result.ok) return result;
const fetchOptions: { headers: Record<string, string>; logger: Logger; signal?: AbortSignal } = {
headers: { Authorization: `Token ${this.token}` },
logger: this.logger,
};
if (options?.signal !== undefined) {
fetchOptions.signal = options.signal;
}
const fetchResult = await fetchWithRetry(url.toString(), fetchOptions);
if (!fetchResult.ok) {
if (fetchResult.error.message.startsWith('HTTP 401')) {
return err(new Error('Invalid API token: authentication failed'));
}
return fetchResult;
}

const parsed = await readJsonCapped(fetchResult.value);
if (!parsed.ok) return parsed;

const data = result.value;
const data = parsed.value;
if (!hasResultsArray(data)) {
return ok([]);
}
Expand All @@ -190,51 +204,5 @@ export class CourtListenerClient {
}
return ok(valid);
}

/** Fetch with exponential backoff retry, including 429 handling */
private async fetchWithRetry(url: string): Promise<Result<unknown>> {
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
const response = await fetch(url, {
headers: { Authorization: `Token ${this.token}` },
});

if (response.ok) {
const parsed = await readJsonCapped(response);
if (!parsed.ok) return parsed;
return ok(parsed.value);
}

if (response.status === 401) {
return err(new Error('Invalid API token: authentication failed'));
}

if (response.status === 429 && attempt < MAX_RETRIES) {
const delayMs = BASE_BACKOFF_MS * Math.pow(2, attempt);
this.logger.warn('Rate limited (429), retrying', { url, attempt, delayMs });
await sleep(delayMs);
continue;
}

if (response.status >= 500 && attempt < MAX_RETRIES) {
const delayMs = BASE_BACKOFF_MS * Math.pow(2, attempt - 1);
this.logger.warn('Server error, retrying', { url, status: response.status, attempt, delayMs });
await sleep(delayMs);
continue;
}

return err(new Error(`HTTP ${response.status}: ${response.statusText}`));
} catch (error: unknown) {
if (attempt < MAX_RETRIES) {
const delayMs = BASE_BACKOFF_MS * Math.pow(2, attempt - 1);
this.logger.warn('Network error, retrying', { url, attempt, delayMs });
await sleep(delayMs);
continue;
}
const message = error instanceof Error ? error.message : String(error);
return err(new Error(`Network error after ${MAX_RETRIES} attempts: ${message}`));
}
}
return err(new Error(`Failed after ${MAX_RETRIES} attempts`));
}
}

Loading