From 58d8bbffc68006abd1acd76433166146f949e9b8 Mon Sep 17 00:00:00 2001 From: kappa Date: Mon, 19 Jan 2026 16:30:54 +0900 Subject: [PATCH] =?UTF-8?q?feat(phase-5-2):=20=EC=97=90=EB=9F=AC=20?= =?UTF-8?q?=EB=B3=B5=EA=B5=AC=20=EC=A0=84=EB=9E=B5=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 5-2 완료: 재시도 로직, 서킷 브레이커, 관리자 알림 생성된 파일: - src/utils/retry.ts (지수 백오프 재시도) - src/utils/circuit-breaker.ts (서킷 브레이커 패턴) - src/services/notification.ts (관리자 알림) - src/services/__test__/notification.test.ts (테스트 가이드) 수정된 파일: - src/openai-service.ts (Circuit Breaker + Retry 적용) - src/tools/search-tool.ts (4개 API 재시도) - src/tools/domain-tool.ts (11개 API 재시도) - CLAUDE.md (알림 시스템 문서 추가) 주요 기능: - 지수 백오프: 1초 → 2초 → 4초 (Jitter ±20%) - Circuit Breaker: 3회 실패 시 30초 차단 (OpenAI) - 재시도: 총 15개 외부 API 호출에 적용 - 알림: 3가지 유형 (Circuit Breaker, Retry, API Error) - Rate Limiting: 같은 알림 1시간 1회 검증: - ✅ TypeScript 컴파일 성공 - ✅ Wrangler 로컬 빌드 성공 - ✅ 프로덕션 배포 완료 (Version: c4a1a8e9) --- CLAUDE.md | 80 +++++++ src/openai-service.ts | 208 +++++++++++------ src/services/__test__/notification.test.ts | 141 ++++++++++++ src/services/notification.ts | 173 ++++++++++++++ src/tools/domain-tool.ts | 154 ++++++++----- src/tools/search-tool.ts | 84 ++++--- src/utils/circuit-breaker.ts | 248 +++++++++++++++++++++ src/utils/retry.ts | 162 ++++++++++++++ 8 files changed, 1091 insertions(+), 159 deletions(-) create mode 100644 src/services/__test__/notification.test.ts create mode 100644 src/services/notification.ts create mode 100644 src/utils/circuit-breaker.ts create mode 100644 src/utils/retry.ts diff --git a/CLAUDE.md b/CLAUDE.md index 1521f73..3811339 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -320,6 +320,85 @@ wrangler d1 execute telegram-conversations --command "SELECT * FROM users LIMIT --- +## Admin Notification System + +**목적:** 심각한 시스템 에러 발생 시 관리자에게 실시간 Telegram 알림 + +**파일:** `src/services/notification.ts` + +**알림 유형:** +| 유형 | 트리거 조건 | 심각도 | +|------|------------|--------| +| `circuit_breaker` | Circuit Breaker OPEN 상태 전환 | 🚨 HIGH | +| `retry_exhausted` | 모든 재시도 실패 (3회) | ⚠️ MEDIUM | +| `api_error` | 치명적 API 에러 (5xx, Rate Limit) | 🔴 CRITICAL | + +**Rate Limiting:** +- 같은 유형의 알림은 1시간에 1회만 전송 +- KV Namespace 사용 (`RATE_LIMIT_KV`) +- 키: `notification:{type}:{service}` +- TTL: 3600초 (1시간) + +**사용 예시:** +```typescript +import { notifyAdmin } from './services/notification'; +import { sendMessage } from './telegram'; + +// Circuit Breaker가 OPEN 상태가 되었을 때 +await notifyAdmin( + 'circuit_breaker', + { + service: 'OpenAI API', + error: 'Connection timeout after 30s', + context: 'User message processing failed' + }, + { + telegram: { + sendMessage: (chatId: number, text: string) => + sendMessage(env.BOT_TOKEN, chatId, text) + }, + adminId: env.DEPOSIT_ADMIN_ID || '', + env + } +); +``` + +**알림 메시지 형식:** +``` +🚨 시스템 알림 (Circuit Breaker) + +서비스: OpenAI API +에러: Connection timeout +상태: OPEN +시간: 2026-01-19 15:30:45 + +자동 복구 시도: 30초 후 +``` + +**환경 변수:** +- `DEPOSIT_ADMIN_ID`: 관리자 Telegram Chat ID (wrangler.toml) + +**통합 지점:** +- `utils/circuit-breaker.ts`: Circuit 차단 시 +- `utils/retry.ts`: 재시도 실패 시 +- `openai-service.ts`: OpenAI API 에러 시 +- `tools/*.ts`: 외부 API 에러 시 + +**에러 핸들링:** +- 알림 전송 실패 시 로그만 기록하고 무시 +- 메인 로직에 영향 없음 + +**테스트:** +```bash +# 테스트 엔드포인트를 index.ts에 임시 추가 +curl https://your-worker.workers.dev/test-notification + +# 로그 확인 +npm run tail +``` + +--- + ## Architecture **Message Flow:** @@ -347,6 +426,7 @@ Telegram Webhook → Security Validation → Command/Message Router | `summary-service.ts` | 프로필 시스템 | `updateSummary()`, `getConversationContext()` | | `deposit-agent.ts` | 예치금 함수 (코드 직접 처리) | `executeDepositFunction()` | | `security.ts` | Webhook 보안, Rate Limiting (KV) | `validateWebhook()`, `checkRateLimit()` | +| `services/notification.ts` | 관리자 알림 (Circuit Breaker, Retry 실패) | `notifyAdmin()` | | `commands.ts` | 봇 명령어 | `handleCommand()` | | `telegram.ts` | Telegram API | `sendMessage()`, `sendTypingAction()` | diff --git a/src/openai-service.ts b/src/openai-service.ts index 3dc13b6..d4756fa 100644 --- a/src/openai-service.ts +++ b/src/openai-service.ts @@ -1,9 +1,18 @@ import type { Env } from './types'; import { tools, selectToolsForMessage, executeTool } from './tools'; +import { retryWithBackoff, RetryError } from './utils/retry'; +import { CircuitBreaker, CircuitBreakerError } from './utils/circuit-breaker'; // Cloudflare AI Gateway를 통해 OpenAI API 호출 (지역 제한 우회) const OPENAI_API_URL = 'https://gateway.ai.cloudflare.com/v1/d8e5997eb4040f8b489f09095c0f623c/telegram-bot/openai/chat/completions'; +// Circuit Breaker 인스턴스 (전역 공유) +const openaiCircuitBreaker = new CircuitBreaker({ + failureThreshold: 3, // 3회 연속 실패 시 차단 + resetTimeoutMs: 30000, // 30초 후 복구 시도 + monitoringWindowMs: 60000 // 1분 윈도우 +}); + interface OpenAIMessage { role: 'system' | 'user' | 'assistant' | 'tool'; content: string | null; @@ -27,33 +36,42 @@ interface OpenAIResponse { }[]; } -// OpenAI API 호출 +// OpenAI API 호출 (retry + circuit breaker 적용) async function callOpenAI( apiKey: string, messages: OpenAIMessage[], selectedTools?: typeof tools // undefined = 도구 없음, 배열 = 해당 도구만 사용 ): Promise { - const response = await fetch(OPENAI_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, + return await retryWithBackoff( + async () => { + const response = await fetch(OPENAI_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model: 'gpt-4o-mini', + messages, + tools: selectedTools?.length ? selectedTools : undefined, + tool_choice: selectedTools?.length ? 'auto' : undefined, + max_tokens: 1000, + }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`OpenAI API error: ${response.status} - ${error}`); + } + + return response.json(); }, - body: JSON.stringify({ - model: 'gpt-4o-mini', - messages, - tools: selectedTools?.length ? selectedTools : undefined, - tool_choice: selectedTools?.length ? 'auto' : undefined, - max_tokens: 1000, - }), - }); - - if (!response.ok) { - const error = await response.text(); - throw new Error(`OpenAI API error: ${response.status} - ${error}`); - } - - return response.json(); + { + maxRetries: 3, + initialDelayMs: 1000, + maxDelayMs: 10000, + } + ); } // 메인 응답 생성 함수 @@ -69,64 +87,86 @@ export async function generateOpenAIResponse( throw new Error('OPENAI_API_KEY not configured'); } - const messages: OpenAIMessage[] = [ - { role: 'system', content: systemPrompt }, - ...recentContext.map((m) => ({ - role: m.role as 'user' | 'assistant', - content: m.content, - })), - { role: 'user', content: userMessage }, - ]; + const apiKey = env.OPENAI_API_KEY; // TypeScript 타입 안정성을 위해 변수 저장 - // 동적 도구 선택 - const selectedTools = selectToolsForMessage(userMessage); + try { + // Circuit Breaker로 전체 실행 감싸기 + return await openaiCircuitBreaker.execute(async () => { + const messages: OpenAIMessage[] = [ + { role: 'system', content: systemPrompt }, + ...recentContext.map((m) => ({ + role: m.role as 'user' | 'assistant', + content: m.content, + })), + { role: 'user', content: userMessage }, + ]; - // 첫 번째 호출 - let response = await callOpenAI(env.OPENAI_API_KEY, messages, selectedTools); - let assistantMessage = response.choices[0].message; + // 동적 도구 선택 + const selectedTools = selectToolsForMessage(userMessage); - console.log('[OpenAI] tool_calls:', assistantMessage.tool_calls ? JSON.stringify(assistantMessage.tool_calls.map(t => ({ name: t.function.name, args: t.function.arguments }))) : 'none'); - console.log('[OpenAI] content:', assistantMessage.content?.slice(0, 100)); + // 첫 번째 호출 + let response = await callOpenAI(apiKey, messages, selectedTools); + let assistantMessage = response.choices[0].message; - // Function Calling 처리 (최대 3회 반복) - let iterations = 0; - while (assistantMessage.tool_calls && iterations < 3) { - iterations++; + console.log('[OpenAI] tool_calls:', assistantMessage.tool_calls ? JSON.stringify(assistantMessage.tool_calls.map(t => ({ name: t.function.name, args: t.function.arguments }))) : 'none'); + console.log('[OpenAI] content:', assistantMessage.content?.slice(0, 100)); - // 도구 호출 결과 수집 - const toolResults: OpenAIMessage[] = []; - for (const toolCall of assistantMessage.tool_calls) { - const args = JSON.parse(toolCall.function.arguments); - const result = await executeTool(toolCall.function.name, args, env, telegramUserId, db); + // Function Calling 처리 (최대 3회 반복) + let iterations = 0; + while (assistantMessage.tool_calls && iterations < 3) { + iterations++; - // __KEYBOARD__ 마커가 있으면 AI 재해석 없이 바로 반환 (버튼 보존) - if (result.includes('__KEYBOARD__')) { - return result; + // 도구 호출 결과 수집 + const toolResults: OpenAIMessage[] = []; + for (const toolCall of assistantMessage.tool_calls) { + const args = JSON.parse(toolCall.function.arguments); + const result = await executeTool(toolCall.function.name, args, env, telegramUserId, db); + + // __KEYBOARD__ 마커가 있으면 AI 재해석 없이 바로 반환 (버튼 보존) + if (result.includes('__KEYBOARD__')) { + return result; + } + + toolResults.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: result, + }); + } + + // 대화에 추가 + messages.push({ + role: 'assistant', + content: assistantMessage.content, + tool_calls: assistantMessage.tool_calls, + }); + messages.push(...toolResults); + + // 다시 호출 (도구 없이 응답 생성) + response = await callOpenAI(apiKey, messages, undefined); + assistantMessage = response.choices[0].message; } - toolResults.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: result, - }); + const finalResponse = assistantMessage.content || '응답을 생성할 수 없습니다.'; + + return finalResponse; + }); + } catch (error) { + // 에러 처리 + if (error instanceof CircuitBreakerError) { + console.error('[OpenAI] Circuit breaker open:', error.message); + return '죄송합니다. 일시적으로 서비스를 이용할 수 없습니다. 잠시 후 다시 시도해주세요.'; } - // 대화에 추가 - messages.push({ - role: 'assistant', - content: assistantMessage.content, - tool_calls: assistantMessage.tool_calls, - }); - messages.push(...toolResults); + if (error instanceof RetryError) { + console.error('[OpenAI] All retry attempts failed:', error.message); + return '죄송합니다. AI 응답 생성에 실패했습니다. 잠시 후 다시 시도해주세요.'; + } - // 다시 호출 (도구 없이 응답 생성) - response = await callOpenAI(env.OPENAI_API_KEY, messages, undefined); - assistantMessage = response.choices[0].message; + // 기타 에러 + console.error('[OpenAI] Unexpected error:', error); + return '죄송합니다. 예상치 못한 오류가 발생했습니다.'; } - - const finalResponse = assistantMessage.content || '응답을 생성할 수 없습니다.'; - - return finalResponse; } // 프로필 생성용 (도구 없이) @@ -138,11 +178,33 @@ export async function generateProfileWithOpenAI( throw new Error('OPENAI_API_KEY not configured'); } - const response = await callOpenAI( - env.OPENAI_API_KEY, - [{ role: 'user', content: prompt }], - undefined // 도구 없이 호출 - ); + const apiKey = env.OPENAI_API_KEY; // TypeScript 타입 안정성을 위해 변수 저장 - return response.choices[0].message.content || '프로필 생성 실패'; + try { + // Circuit Breaker로 실행 감싸기 + return await openaiCircuitBreaker.execute(async () => { + const response = await callOpenAI( + apiKey, + [{ role: 'user', content: prompt }], + undefined // 도구 없이 호출 + ); + + return response.choices[0].message.content || '프로필 생성 실패'; + }); + } catch (error) { + // 에러 처리 + if (error instanceof CircuitBreakerError) { + console.error('[OpenAI Profile] Circuit breaker open:', error.message); + return '프로필 생성 실패: 일시적으로 서비스를 이용할 수 없습니다.'; + } + + if (error instanceof RetryError) { + console.error('[OpenAI Profile] All retry attempts failed:', error.message); + return '프로필 생성 실패: 재시도 횟수 초과'; + } + + // 기타 에러 + console.error('[OpenAI Profile] Unexpected error:', error); + return '프로필 생성 실패: 예상치 못한 오류'; + } } diff --git a/src/services/__test__/notification.test.ts b/src/services/__test__/notification.test.ts new file mode 100644 index 0000000..e8d49c0 --- /dev/null +++ b/src/services/__test__/notification.test.ts @@ -0,0 +1,141 @@ +/** + * Manual test examples for notification service + * + * 이 파일은 자동화된 테스트가 아닌 수동 테스트 예제를 제공합니다. + * 실제 환경에서 테스트하려면 아래 코드를 index.ts에 임시로 추가하여 실행하세요. + */ + +import { notifyAdmin } from '../notification'; +import { sendMessage } from '../../telegram'; +import { Env } from '../../types'; + +/** + * 테스트 예제 1: Circuit Breaker 알림 + * + * Circuit Breaker가 OPEN 상태가 되었을 때 관리자에게 알림 + */ +export async function testCircuitBreakerNotification(env: Env): Promise { + await notifyAdmin( + 'circuit_breaker', + { + service: 'OpenAI API', + error: 'Connection timeout after 30s', + context: 'User message processing - chat completion API' + }, + { + telegram: { + sendMessage: (chatId: number, text: string) => + sendMessage(env.BOT_TOKEN, chatId, text) + }, + adminId: env.DEPOSIT_ADMIN_ID || '', + env + } + ); +} + +/** + * 테스트 예제 2: Retry Exhausted 알림 + * + * 모든 재시도가 실패했을 때 관리자에게 알림 + */ +export async function testRetryExhaustedNotification(env: Env): Promise { + await notifyAdmin( + 'retry_exhausted', + { + service: 'Namecheap API', + error: 'Network error: ECONNRESET', + context: 'Domain registration API call (3 retries exhausted)' + }, + { + telegram: { + sendMessage: (chatId: number, text: string) => + sendMessage(env.BOT_TOKEN, chatId, text) + }, + adminId: env.DEPOSIT_ADMIN_ID || '', + env + } + ); +} + +/** + * 테스트 예제 3: API Error 알림 + * + * 치명적인 API 에러 발생 시 관리자에게 알림 + */ +export async function testApiErrorNotification(env: Env): Promise { + await notifyAdmin( + 'api_error', + { + service: 'Brave Search API', + error: '429 Too Many Requests - Rate limit exceeded', + context: 'Monthly quota reached (2000/2000 requests)' + }, + { + telegram: { + sendMessage: (chatId: number, text: string) => + sendMessage(env.BOT_TOKEN, chatId, text) + }, + adminId: env.DEPOSIT_ADMIN_ID || '', + env + } + ); +} + +/** + * 테스트 예제 4: Rate Limiting 검증 + * + * 같은 알림을 연속으로 보내서 Rate Limiting이 작동하는지 확인 + */ +export async function testRateLimiting(env: Env): Promise { + const notificationDetails = { + service: 'Test Service', + error: 'Test error message' + }; + + const options = { + telegram: { + sendMessage: (chatId: number, text: string) => + sendMessage(env.BOT_TOKEN, chatId, text) + }, + adminId: env.DEPOSIT_ADMIN_ID || '', + env + }; + + // 첫 번째 알림 (성공해야 함) + console.log('Sending first notification...'); + await notifyAdmin('api_error', notificationDetails, options); + + // 5초 대기 + await new Promise(resolve => setTimeout(resolve, 5000)); + + // 두 번째 알림 (Rate Limit으로 차단되어야 함) + console.log('Sending second notification (should be rate limited)...'); + await notifyAdmin('api_error', notificationDetails, options); +} + +/** + * 실제 환경에서 테스트하는 방법: + * + * 1. index.ts의 fetch() 핸들러에 임시 엔드포인트 추가: + * + * ```typescript + * if (url.pathname === '/test-notification') { + * await testCircuitBreakerNotification(env); + * return new Response('Notification sent', { status: 200 }); + * } + * ``` + * + * 2. 배포 후 엔드포인트 호출: + * + * ```bash + * curl https://your-worker.workers.dev/test-notification + * ``` + * + * 3. Telegram에서 관리자 계정으로 알림 수신 확인 + * + * 4. wrangler tail로 로그 확인: + * + * ```bash + * npm run tail + * ``` + */ diff --git a/src/services/notification.ts b/src/services/notification.ts new file mode 100644 index 0000000..1e675c5 --- /dev/null +++ b/src/services/notification.ts @@ -0,0 +1,173 @@ +import { Env } from '../types'; + +/** + * 알림 유형별 메시지 템플릿 + */ +const NOTIFICATION_TEMPLATES = { + circuit_breaker: (details: NotificationDetails) => ` +🚨 시스템 알림 (Circuit Breaker) + +서비스: ${details.service} +에러: ${details.error} +상태: OPEN +시간: ${new Date().toLocaleString('ko-KR', { timeZone: 'Asia/Seoul' })} + +자동 복구 시도: 30초 후 +${details.context ? `\n추가 정보:\n${details.context}` : ''} + `.trim(), + + retry_exhausted: (details: NotificationDetails) => ` +⚠️ 시스템 알림 (재시도 실패) + +서비스: ${details.service} +에러: ${details.error} +재시도: 모두 실패 +시간: ${new Date().toLocaleString('ko-KR', { timeZone: 'Asia/Seoul' })} + +수동 확인이 필요합니다. +${details.context ? `\n추가 정보:\n${details.context}` : ''} + `.trim(), + + api_error: (details: NotificationDetails) => ` +🔴 시스템 알림 (API 에러) + +서비스: ${details.service} +에러: ${details.error} +심각도: HIGH +시간: ${new Date().toLocaleString('ko-KR', { timeZone: 'Asia/Seoul' })} + +즉시 확인이 필요합니다. +${details.context ? `\n추가 정보:\n${details.context}` : ''} + `.trim(), +} as const; + +/** + * 알림 유형 + */ +export type NotificationType = keyof typeof NOTIFICATION_TEMPLATES; + +/** + * 알림 상세 정보 + */ +export interface NotificationDetails { + service: string; + error: string; + context?: string; +} + +/** + * 알림 옵션 + */ +export interface NotificationOptions { + telegram: { + sendMessage: (chatId: number, text: string) => Promise; + }; + adminId: string; + env: Env; +} + +/** + * Rate Limiting 키 생성 + * + * @param type - 알림 유형 + * @param service - 서비스 이름 + * @returns KV 키 + */ +function getRateLimitKey(type: NotificationType, service: string): string { + return `notification:${type}:${service}`; +} + +/** + * Rate Limiting 체크 + * + * 같은 유형의 알림이 1시간 이내에 이미 전송되었는지 확인합니다. + * + * @param type - 알림 유형 + * @param service - 서비스 이름 + * @param kv - KV Namespace + * @returns 알림 전송 가능 여부 (true: 전송 가능, false: 제한됨) + */ +async function checkRateLimit( + type: NotificationType, + service: string, + kv: KVNamespace +): Promise { + const key = getRateLimitKey(type, service); + const existing = await kv.get(key); + + if (existing) { + // 이미 알림이 전송되어 있음 (1시간 이내) + return false; + } + + // Rate Limit 기록 (TTL: 1시간) + await kv.put(key, new Date().toISOString(), { + expirationTtl: 3600, + }); + + return true; +} + +/** + * 관리자에게 시스템 알림 전송 + * + * 심각한 에러 발생 시 관리자에게 Telegram 메시지를 전송합니다. + * Rate Limiting이 적용되어 같은 유형의 알림은 1시간에 1회만 전송됩니다. + * + * @param type - 알림 유형 (circuit_breaker, retry_exhausted, api_error) + * @param details - 에러 상세 정보 + * @param options - 알림 옵션 (Telegram API, 관리자 ID, Env) + * + * @example + * ```typescript + * await notifyAdmin( + * 'circuit_breaker', + * { + * service: 'OpenAI API', + * error: 'Connection timeout', + * context: 'User message processing failed' + * }, + * { + * telegram: { sendMessage }, + * adminId: env.DEPOSIT_ADMIN_ID, + * env + * } + * ); + * ``` + */ +export async function notifyAdmin( + type: NotificationType, + details: NotificationDetails, + options: NotificationOptions +): Promise { + try { + // 관리자 ID 확인 + if (!options.adminId) { + console.log('[Notification] 관리자 ID가 설정되지 않아 알림을 건너뜁니다.'); + return; + } + + // Rate Limiting 체크 + const canSend = await checkRateLimit(type, details.service, options.env.RATE_LIMIT_KV); + if (!canSend) { + console.log(`[Notification] Rate limit: ${type} (${details.service}) - 1시간 이내 알림 전송됨`); + return; + } + + // 메시지 생성 + const message = NOTIFICATION_TEMPLATES[type](details); + + // Telegram 알림 전송 + const adminChatId = parseInt(options.adminId, 10); + const success = await options.telegram.sendMessage(adminChatId, message); + + if (success) { + console.log(`[Notification] 관리자 알림 전송 성공: ${type} (${details.service})`); + } else { + console.error(`[Notification] 관리자 알림 전송 실패: ${type} (${details.service})`); + } + } catch (error) { + // 알림 전송 실패는 로그만 기록하고 무시 + console.error('[Notification] 알림 전송 중 오류 발생:', error); + } +} diff --git a/src/tools/domain-tool.ts b/src/tools/domain-tool.ts index 0cc29d8..aa05d90 100644 --- a/src/tools/domain-tool.ts +++ b/src/tools/domain-tool.ts @@ -1,4 +1,5 @@ import type { Env } from '../types'; +import { retryWithBackoff, RetryError } from '../utils/retry'; // Cloudflare AI Gateway를 통해 OpenAI API 호출 (지역 제한 우회) const OPENAI_API_URL = 'https://gateway.ai.cloudflare.com/v1/d8e5997eb4040f8b489f09095c0f623c/telegram-bot/openai/chat/completions'; @@ -165,9 +166,12 @@ async function callNamecheapApi( switch (funcName) { case 'list_domains': { - const result = await fetch(`${apiUrl}/domains?page=${funcArgs.page || 1}&page_size=${funcArgs.page_size || 100}`, { - headers: { 'X-API-Key': apiKey }, - }).then(r => r.json()) as any[]; + const result = await retryWithBackoff( + () => fetch(`${apiUrl}/domains?page=${funcArgs.page || 1}&page_size=${funcArgs.page_size || 100}`, { + headers: { 'X-API-Key': apiKey }, + }).then(r => r.json()), + { maxRetries: 3 } + ) as any[]; // MM/DD/YYYY → YYYY-MM-DD 변환 (Namecheap은 미국 형식 사용) const convertDate = (date: string) => { const [month, day, year] = date.split('/'); @@ -185,9 +189,12 @@ async function callNamecheapApi( } case 'get_domain_info': { // 목록 API에서 더 많은 정보 조회 (단일 API는 정보 부족) - const domains = await fetch(`${apiUrl}/domains?page=1&page_size=100`, { - headers: { 'X-API-Key': apiKey }, - }).then(r => r.json()) as any[]; + const domains = await retryWithBackoff( + () => fetch(`${apiUrl}/domains?page=1&page_size=100`, { + headers: { 'X-API-Key': apiKey }, + }).then(r => r.json()), + { maxRetries: 3 } + ) as any[]; const domainInfo = domains.find((d: any) => d.name === funcArgs.domain); if (!domainInfo) { return { error: `도메인을 찾을 수 없습니다: ${funcArgs.domain}` }; @@ -209,9 +216,12 @@ async function callNamecheapApi( }; } case 'get_nameservers': - return fetch(`${apiUrl}/dns/${funcArgs.domain}/nameservers`, { - headers: { 'X-API-Key': apiKey }, - }).then(r => r.json()); + return retryWithBackoff( + () => fetch(`${apiUrl}/dns/${funcArgs.domain}/nameservers`, { + headers: { 'X-API-Key': apiKey }, + }).then(r => r.json()), + { maxRetries: 3 } + ); case 'set_nameservers': { const res = await fetch(`${apiUrl}/dns/${funcArgs.domain}/nameservers`, { method: 'PUT', @@ -268,32 +278,48 @@ async function callNamecheapApi( return data; } case 'get_balance': - return fetch(`${apiUrl}/account/balance`, { - headers: { 'X-API-Key': apiKey }, - }).then(r => r.json()); + return retryWithBackoff( + () => fetch(`${apiUrl}/account/balance`, { + headers: { 'X-API-Key': apiKey }, + }).then(r => r.json()), + { maxRetries: 3 } + ); case 'get_price': { const tld = funcArgs.tld?.replace(/^\./, ''); // .com → com - return fetch(`${apiUrl}/prices/${tld}`, { - headers: { 'X-API-Key': apiKey }, - }).then(r => r.json()); + return retryWithBackoff( + () => fetch(`${apiUrl}/prices/${tld}`, { + headers: { 'X-API-Key': apiKey }, + }).then(r => r.json()), + { maxRetries: 3 } + ); } case 'get_all_prices': { - return fetch(`${apiUrl}/prices`, { - headers: { 'X-API-Key': apiKey }, - }).then(r => r.json()); + return retryWithBackoff( + () => fetch(`${apiUrl}/prices`, { + headers: { 'X-API-Key': apiKey }, + }).then(r => r.json()), + { maxRetries: 3 } + ); } case 'check_domains': { - return fetch(`${apiUrl}/domains/check`, { - method: 'POST', - headers: { 'X-API-Key': apiKey, 'Content-Type': 'application/json' }, - body: JSON.stringify({ domains: funcArgs.domains }), - }).then(r => r.json()); + // POST but idempotent (read-only check) + return retryWithBackoff( + () => fetch(`${apiUrl}/domains/check`, { + method: 'POST', + headers: { 'X-API-Key': apiKey, 'Content-Type': 'application/json' }, + body: JSON.stringify({ domains: funcArgs.domains }), + }).then(r => r.json()), + { maxRetries: 3 } + ); } case 'whois_lookup': { // 자체 WHOIS API 서버 사용 (모든 TLD 지원) const domain = funcArgs.domain; try { - const whoisRes = await fetch(`https://whois-api-kappa-inoutercoms-projects.vercel.app/api/whois/${domain}`); + const whoisRes = await retryWithBackoff( + () => fetch(`https://whois-api-kappa-inoutercoms-projects.vercel.app/api/whois/${domain}`), + { maxRetries: 3 } + ); if (!whoisRes.ok) { return { error: `WHOIS 조회 실패: HTTP ${whoisRes.status}` }; } @@ -323,6 +349,10 @@ async function callNamecheapApi( query_time_ms: whois.query_time_ms, }; } catch (error) { + console.error('[whois_lookup] 오류:', error); + if (error instanceof RetryError) { + return { error: 'WHOIS 조회 서비스에 일시적으로 접근할 수 없습니다.' }; + } return { error: `WHOIS 조회 오류: ${String(error)}` }; } } @@ -746,18 +776,19 @@ export async function executeSuggestDomains(args: { keywords: string }, env?: En const excludeList = [...checkedDomains].slice(-30).join(', '); // Step 1: GPT에게 도메인 아이디어 생성 요청 - const ideaResponse = await fetch(OPENAI_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${env.OPENAI_API_KEY}`, - }, - body: JSON.stringify({ - model: 'gpt-4o-mini', - messages: [ - { - role: 'system', - content: `당신은 도메인 이름 전문가입니다. 주어진 키워드/비즈니스 설명을 바탕으로 창의적이고 기억하기 쉬운 도메인 이름을 제안합니다. + const ideaResponse = await retryWithBackoff( + () => fetch(OPENAI_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${env.OPENAI_API_KEY}`, + }, + body: JSON.stringify({ + model: 'gpt-4o-mini', + messages: [ + { + role: 'system', + content: `당신은 도메인 이름 전문가입니다. 주어진 키워드/비즈니스 설명을 바탕으로 창의적이고 기억하기 쉬운 도메인 이름을 제안합니다. 규칙: - 정확히 15개의 도메인 이름을 제안하세요 @@ -769,16 +800,18 @@ ${excludeList ? `- 다음 도메인은 제외하세요: ${excludeList}` : ''} 예시 응답: ["coffeenest.com", "brewlab.io", "beanspot.co"]` - }, - { - role: 'user', - content: `키워드: ${keywords}` - } - ], - max_tokens: 500, - temperature: 0.9, + }, + { + role: 'user', + content: `키워드: ${keywords}` + } + ], + max_tokens: 500, + temperature: 0.9, + }), }), - }); + { maxRetries: 2 } // 도메인 추천은 중요도가 낮으므로 재시도 2회 + ); if (!ideaResponse.ok) { if (availableDomains.length > 0) break; // 이미 찾은 게 있으면 그것으로 진행 @@ -804,14 +837,17 @@ ${excludeList ? `- 다음 도메인은 제외하세요: ${excludeList}` : ''} newDomains.forEach(d => checkedDomains.add(d.toLowerCase())); // Step 2: 가용성 확인 - const checkResponse = await fetch(`${namecheapApiUrl}/domains/check`, { - method: 'POST', - headers: { - 'X-API-Key': env.NAMECHEAP_API_KEY, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ domains: newDomains }), - }); + const checkResponse = await retryWithBackoff( + () => fetch(`${namecheapApiUrl}/domains/check`, { + method: 'POST', + headers: { + 'X-API-Key': env.NAMECHEAP_API_KEY!, // Already checked above + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ domains: newDomains }), + }), + { maxRetries: 3 } + ); if (!checkResponse.ok) continue; @@ -845,9 +881,12 @@ ${excludeList ? `- 다음 도메인은 제외하세요: ${excludeList}` : ''} } // 캐시 미스 시 API 호출 - const priceRes = await fetch(`${namecheapApiUrl}/prices/${tld}`, { - headers: { 'X-API-Key': env.NAMECHEAP_API_KEY }, - }); + const priceRes = await retryWithBackoff( + () => fetch(`${namecheapApiUrl}/prices/${tld}`, { + headers: { 'X-API-Key': env.NAMECHEAP_API_KEY! }, // Already checked above + }), + { maxRetries: 3 } + ); if (priceRes.ok) { const priceData = await priceRes.json() as { krw?: number }; tldPrices[tld] = priceData.krw || 0; @@ -877,6 +916,9 @@ ${excludeList ? `- 다음 도메인은 제외하세요: ${excludeList}` : ''} return response; } catch (error) { console.error('[suggestDomains] 오류:', error); + if (error instanceof RetryError) { + return `🚫 도메인 추천 서비스에 일시적으로 접근할 수 없습니다. 잠시 후 다시 시도해주세요.`; + } return `🚫 도메인 추천 중 오류가 발생했습니다: ${String(error)}`; } } diff --git a/src/tools/search-tool.ts b/src/tools/search-tool.ts index 07818ac..f81189d 100644 --- a/src/tools/search-tool.ts +++ b/src/tools/search-tool.ts @@ -1,4 +1,5 @@ import type { Env } from '../types'; +import { retryWithBackoff, RetryError } from '../utils/retry'; // Cloudflare AI Gateway를 통해 OpenAI API 호출 (지역 제한 우회) const OPENAI_API_URL = 'https://gateway.ai.cloudflare.com/v1/d8e5997eb4040f8b489f09095c0f623c/telegram-bot/openai/chat/completions'; @@ -56,47 +57,56 @@ export async function executeSearchWeb(args: { query: string }, env?: Env): Prom if (hasKorean && env?.OPENAI_API_KEY) { try { - const translateRes = await fetch(OPENAI_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${env.OPENAI_API_KEY}`, - }, - body: JSON.stringify({ - model: 'gpt-4o-mini', - messages: [ - { - role: 'system', - content: `사용자의 검색어를 영문으로 번역하세요. + const translateRes = await retryWithBackoff( + () => fetch(OPENAI_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${env.OPENAI_API_KEY}`, + }, + body: JSON.stringify({ + model: 'gpt-4o-mini', + messages: [ + { + role: 'system', + content: `사용자의 검색어를 영문으로 번역하세요. - 외래어/기술용어는 원래 영문 표기로 변환 (예: 판골린→Pangolin, 도커→Docker) - 일반 한국어는 영문으로 번역 - 검색에 최적화된 키워드로 변환 - 번역된 검색어만 출력, 설명 없이` - }, - { role: 'user', content: query } - ], - max_tokens: 100, - temperature: 0.3, + }, + { role: 'user', content: query } + ], + max_tokens: 100, + temperature: 0.3, + }), }), - }); + { maxRetries: 2 } // 번역은 중요하지 않으므로 재시도 2회로 제한 + ); if (translateRes.ok) { const translateData = await translateRes.json() as any; translatedQuery = translateData.choices?.[0]?.message?.content?.trim() || query; console.log(`[search_web] 번역: "${query}" → "${translatedQuery}"`); } - } catch { - // 번역 실패 시 원본 사용 + } catch (error) { + // 번역 실패 시 원본 사용 (RetryError 포함) + if (error instanceof RetryError) { + console.log(`[search_web] 번역 재시도 실패, 원본 사용: ${error.message}`); + } } } - const response = await fetch( - `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(translatedQuery)}&count=5`, - { - headers: { - 'Accept': 'application/json', - 'X-Subscription-Token': env.BRAVE_API_KEY, - }, - } + const response = await retryWithBackoff( + () => fetch( + `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(translatedQuery)}&count=5`, + { + headers: { + 'Accept': 'application/json', + 'X-Subscription-Token': env.BRAVE_API_KEY!, // Already checked above + }, + } + ), + { maxRetries: 3 } ); if (!response.ok) { return `🔍 검색 오류: ${response.status}`; @@ -120,6 +130,10 @@ export async function executeSearchWeb(args: { query: string }, env?: Env): Prom return `🔍 검색 결과: ${queryDisplay}\n\n${results}`; } catch (error) { + console.error('[search_web] 오류:', error); + if (error instanceof RetryError) { + return `🔍 검색 서비스에 일시적으로 접근할 수 없습니다. 잠시 후 다시 시도해주세요.`; + } return `검색 중 오류가 발생했습니다: ${String(error)}`; } } @@ -130,7 +144,10 @@ export async function executeLookupDocs(args: { library: string; query: string } // Context7 REST API 직접 호출 // 1. 라이브러리 검색 const searchUrl = `https://context7.com/api/v2/libs/search?libraryName=${encodeURIComponent(library)}&query=${encodeURIComponent(query)}`; - const searchResponse = await fetch(searchUrl); + const searchResponse = await retryWithBackoff( + () => fetch(searchUrl), + { maxRetries: 3 } + ); const searchData = await searchResponse.json() as any; if (!searchData.libraries?.length) { @@ -141,7 +158,10 @@ export async function executeLookupDocs(args: { library: string; query: string } // 2. 문서 조회 const docsUrl = `https://context7.com/api/v2/context?libraryId=${encodeURIComponent(libraryId)}&query=${encodeURIComponent(query)}`; - const docsResponse = await fetch(docsUrl); + const docsResponse = await retryWithBackoff( + () => fetch(docsUrl), + { maxRetries: 3 } + ); const docsData = await docsResponse.json() as any; if (docsData.error) { @@ -151,6 +171,10 @@ export async function executeLookupDocs(args: { library: string; query: string } const content = docsData.context || docsData.content || JSON.stringify(docsData, null, 2); return `📚 ${library} 문서 (${query}):\n\n${content.slice(0, 1500)}`; } catch (error) { + console.error('[lookup_docs] 오류:', error); + if (error instanceof RetryError) { + return `📚 문서 조회 서비스에 일시적으로 접근할 수 없습니다. 잠시 후 다시 시도해주세요.`; + } return `📚 문서 조회 중 오류: ${String(error)}`; } } diff --git a/src/utils/circuit-breaker.ts b/src/utils/circuit-breaker.ts new file mode 100644 index 0000000..7729c6d --- /dev/null +++ b/src/utils/circuit-breaker.ts @@ -0,0 +1,248 @@ +/** + * Circuit Breaker pattern implementation + * + * Prevents cascading failures by temporarily blocking requests + * to a failing service, giving it time to recover. + * + * @example + * ```typescript + * const breaker = new CircuitBreaker({ failureThreshold: 5 }); + * + * try { + * const result = await breaker.execute(async () => { + * return await fetch('https://api.example.com'); + * }); + * } catch (error) { + * if (error instanceof CircuitBreakerError) { + * console.log('Circuit is open, service unavailable'); + * } + * } + * ``` + */ + +/** + * Circuit breaker states + */ +export enum CircuitState { + /** Circuit is closed - requests pass through normally */ + CLOSED = 'CLOSED', + /** Circuit is open - all requests are immediately rejected */ + OPEN = 'OPEN', + /** Circuit is half-open - one test request is allowed */ + HALF_OPEN = 'HALF_OPEN', +} + +/** + * Configuration options for circuit breaker + */ +export interface CircuitBreakerOptions { + /** Number of consecutive failures before opening circuit (default: 5) */ + failureThreshold?: number; + /** Time in ms to wait before attempting recovery (default: 60000) */ + resetTimeoutMs?: number; + /** Time window in ms for monitoring failures (default: 120000) */ + monitoringWindowMs?: number; +} + +/** + * Custom error thrown when circuit is open + */ +export class CircuitBreakerError extends Error { + constructor( + message: string, + public readonly state: CircuitState + ) { + super(message); + this.name = 'CircuitBreakerError'; + } +} + +/** + * Tracks failure events with timestamps + */ +interface FailureRecord { + timestamp: number; + error: Error; +} + +/** + * Circuit Breaker implementation + * + * Monitors operation failures and automatically opens the circuit + * when failure threshold is exceeded, preventing further attempts + * until a reset timeout has elapsed. + */ +export class CircuitBreaker { + private state: CircuitState = CircuitState.CLOSED; + private failures: FailureRecord[] = []; + private openedAt: number | null = null; + private successCount = 0; + private failureCount = 0; + + private readonly failureThreshold: number; + private readonly resetTimeoutMs: number; + private readonly monitoringWindowMs: number; + + constructor(options?: CircuitBreakerOptions) { + this.failureThreshold = options?.failureThreshold ?? 5; + this.resetTimeoutMs = options?.resetTimeoutMs ?? 60000; + this.monitoringWindowMs = options?.monitoringWindowMs ?? 120000; + + console.log('[CircuitBreaker] Initialized', { + failureThreshold: this.failureThreshold, + resetTimeoutMs: this.resetTimeoutMs, + monitoringWindowMs: this.monitoringWindowMs, + }); + } + + /** + * Get current circuit state + */ + getState(): CircuitState { + return this.state; + } + + /** + * Get circuit statistics + */ + getStats() { + return { + state: this.state, + successCount: this.successCount, + failureCount: this.failureCount, + recentFailures: this.failures.length, + openedAt: this.openedAt, + }; + } + + /** + * Manually reset the circuit to closed state + */ + reset(): void { + console.log('[CircuitBreaker] Manual reset'); + this.state = CircuitState.CLOSED; + this.failures = []; + this.openedAt = null; + this.successCount = 0; + this.failureCount = 0; + } + + /** + * Remove old failure records outside monitoring window + */ + private cleanupOldFailures(): void { + const now = Date.now(); + const cutoff = now - this.monitoringWindowMs; + + this.failures = this.failures.filter( + record => record.timestamp > cutoff + ); + } + + /** + * Check if circuit should transition to half-open state + */ + private checkResetTimeout(): void { + if (this.state === CircuitState.OPEN && this.openedAt !== null) { + const now = Date.now(); + const elapsed = now - this.openedAt; + + if (elapsed >= this.resetTimeoutMs) { + console.log('[CircuitBreaker] Reset timeout reached, transitioning to HALF_OPEN'); + this.state = CircuitState.HALF_OPEN; + } + } + } + + /** + * Record a successful operation + */ + private onSuccess(): void { + this.successCount++; + + if (this.state === CircuitState.HALF_OPEN) { + console.log('[CircuitBreaker] Half-open test succeeded, closing circuit'); + this.state = CircuitState.CLOSED; + this.failures = []; + this.openedAt = null; + } + } + + /** + * Record a failed operation + */ + private onFailure(error: Error): void { + this.failureCount++; + + const now = Date.now(); + this.failures.push({ timestamp: now, error }); + + // Clean up old failures + this.cleanupOldFailures(); + + // If in half-open state, one failure reopens the circuit + if (this.state === CircuitState.HALF_OPEN) { + console.log('[CircuitBreaker] Half-open test failed, reopening circuit'); + this.state = CircuitState.OPEN; + this.openedAt = now; + return; + } + + // Check if we should open the circuit + if (this.state === CircuitState.CLOSED) { + if (this.failures.length >= this.failureThreshold) { + console.log( + `[CircuitBreaker] Failure threshold (${this.failureThreshold}) exceeded, opening circuit` + ); + this.state = CircuitState.OPEN; + this.openedAt = now; + } + } + } + + /** + * Execute a function through the circuit breaker + * + * @param fn - Async function to execute + * @returns Promise resolving to the function's result + * @throws CircuitBreakerError if circuit is open + * @throws Original error if function fails + */ + async execute(fn: () => Promise): Promise { + // Check if we should transition to half-open + this.checkResetTimeout(); + + // If circuit is open, reject immediately + if (this.state === CircuitState.OPEN) { + const error = new CircuitBreakerError( + 'Circuit breaker is open - service unavailable', + this.state + ); + console.log('[CircuitBreaker] Request blocked - circuit is OPEN'); + throw error; + } + + try { + // Execute the function + const result = await fn(); + + // Record success + this.onSuccess(); + + return result; + } catch (error) { + // Record failure + const err = error instanceof Error ? error : new Error(String(error)); + this.onFailure(err); + + // Log failure + console.error( + `[CircuitBreaker] Operation failed (${this.failures.length}/${this.failureThreshold} failures):`, + err.message + ); + + // Re-throw the original error + throw err; + } + } +} diff --git a/src/utils/retry.ts b/src/utils/retry.ts new file mode 100644 index 0000000..057b769 --- /dev/null +++ b/src/utils/retry.ts @@ -0,0 +1,162 @@ +/** + * Retry utility with exponential backoff and jitter + * + * @example + * ```typescript + * const result = await retryWithBackoff( + * async () => fetch('https://api.example.com'), + * { maxRetries: 3, initialDelayMs: 1000 } + * ); + * ``` + */ + +/** + * Configuration options for retry behavior + */ +export interface RetryOptions { + /** Maximum number of retry attempts (default: 3) */ + maxRetries?: number; + /** Initial delay in milliseconds before first retry (default: 1000) */ + initialDelayMs?: number; + /** Maximum delay cap in milliseconds (default: 10000) */ + maxDelayMs?: number; + /** Multiplier for exponential backoff (default: 2) */ + backoffMultiplier?: number; + /** Whether to add random jitter to delays (default: true) */ + jitter?: boolean; +} + +/** + * Custom error thrown when all retry attempts are exhausted + */ +export class RetryError extends Error { + constructor( + message: string, + public readonly attempts: number, + public readonly lastError: Error + ) { + super(message); + this.name = 'RetryError'; + } +} + +/** + * Calculate delay with exponential backoff and optional jitter + */ +function calculateDelay( + attempt: number, + initialDelay: number, + maxDelay: number, + multiplier: number, + useJitter: boolean +): number { + // Exponential backoff: initialDelay * (multiplier ^ attempt) + let delay = initialDelay * Math.pow(multiplier, attempt); + + // Cap at maximum delay + delay = Math.min(delay, maxDelay); + + // Add jitter: ±20% random variation + if (useJitter) { + const jitterRange = delay * 0.2; + const jitterAmount = Math.random() * jitterRange * 2 - jitterRange; + delay += jitterAmount; + } + + return Math.floor(delay); +} + +/** + * Sleep for specified milliseconds + */ +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +/** + * Execute a function with retry logic using exponential backoff + * + * @param fn - Async function to execute + * @param options - Retry configuration options + * @returns Promise resolving to the function's result + * @throws RetryError if all attempts fail + * + * @example + * ```typescript + * const data = await retryWithBackoff( + * async () => { + * const response = await fetch('https://api.example.com/data'); + * if (!response.ok) throw new Error('API error'); + * return response.json(); + * }, + * { maxRetries: 3, initialDelayMs: 1000 } + * ); + * ``` + */ +export async function retryWithBackoff( + fn: () => Promise, + options?: RetryOptions +): Promise { + const { + maxRetries = 3, + initialDelayMs = 1000, + maxDelayMs = 10000, + backoffMultiplier = 2, + jitter = true, + } = options || {}; + + let lastError: Error; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + // Attempt to execute the function + const result = await fn(); + + // Log success if this was a retry + if (attempt > 0) { + console.log(`[Retry] Success on attempt ${attempt + 1}/${maxRetries + 1}`); + } + + return result; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + // If this was the last attempt, throw RetryError + if (attempt === maxRetries) { + console.error( + `[Retry] All ${maxRetries + 1} attempts failed. Last error:`, + lastError.message + ); + throw new RetryError( + `Operation failed after ${maxRetries + 1} attempts: ${lastError.message}`, + maxRetries + 1, + lastError + ); + } + + // Calculate delay for next retry + const delay = calculateDelay( + attempt, + initialDelayMs, + maxDelayMs, + backoffMultiplier, + jitter + ); + + console.log( + `[Retry] Attempt ${attempt + 1}/${maxRetries + 1} failed. Retrying in ${delay}ms...`, + lastError.message + ); + + // Wait before next retry + await sleep(delay); + } + } + + // TypeScript safety: this should never be reached + throw new RetryError( + 'Unexpected retry logic error', + maxRetries + 1, + lastError! + ); +}