chunzhi/comfyui-deploy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add a init timeout of 10 seconds for retry logic

Browse Source
This commit is contained in:
bennykok 2024-08-18 17:31:48 -07:00
parent dd1d9df23f
commit 507d5ef631
1 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
custom_routes.py
View File

@ -23,7 +23,7 @@ from typing import Dict, List, Union, Any, Optional
from PIL import Image from PIL import Image
import copy import copy
import struct import struct
from aiohttp import web, ClientSession, ClientError from aiohttp import web, ClientSession, ClientError, ClientTimeout
import atexit import atexit
# Global session # Global session
@ -51,7 +51,7 @@ def exit_handler():
atexit.register(exit_handler) atexit.register(exit_handler)
max_retries = int(os.environ.get('MAX_RETRIES', '3')) max_retries = int(os.environ.get('MAX_RETRIES', '5'))
retry_delay_multiplier = float(os.environ.get('RETRY_DELAY_MULTIPLIER', '2')) retry_delay_multiplier = float(os.environ.get('RETRY_DELAY_MULTIPLIER', '2'))
print(f"max_retries: {max_retries}, retry_delay_multiplier: {retry_delay_multiplier}") print(f"max_retries: {max_retries}, retry_delay_multiplier: {retry_delay_multiplier}")
@ -60,19 +60,31 @@ async def async_request_with_retry(method, url, **kwargs):
global client_session global client_session
await ensure_client_session() await ensure_client_session()
retry_delay = 1 # Start with 1 second delay retry_delay = 1 # Start with 1 second delay
initial_timeout = 5 # 5 seconds timeout for the initial connection
for attempt in range(max_retries): for attempt in range(max_retries):
try: try:
# Set a timeout for the initial connection
timeout = ClientTimeout(total=None, connect=initial_timeout)
kwargs['timeout'] = timeout
async with client_session.request(method, url, **kwargs) as response: async with client_session.request(method, url, **kwargs) as response:
response.raise_for_status() response.raise_for_status()
return response return response
except asyncio.TimeoutError:
logger.warning(f"Request timed out after {initial_timeout} seconds (attempt {attempt + 1}/{max_retries})")
except ClientError as e: except ClientError as e:
if attempt == max_retries - 1: if attempt == max_retries - 1:
logger.error(f"Request failed after {max_retries} attempts: {e}") logger.error(f"Request failed after {max_retries} attempts: {e}")
# raise # raise
logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}") logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}")
await asyncio.sleep(retry_delay)
retry_delay *= retry_delay_multiplier # Exponential backoff # Wait before retrying
await asyncio.sleep(retry_delay)
retry_delay *= retry_delay_multiplier # Exponential backoff
# If all retries fail, raise an exception
raise Exception(f"Request failed after {max_retries} attempts")
from logging import basicConfig, getLogger from logging import basicConfig, getLogger