diff --git a/builder/modal-builder/src/main.py b/builder/modal-builder/src/main.py index 3381573..a63e379 100644 --- a/builder/modal-builder/src/main.py +++ b/builder/modal-builder/src/main.py @@ -180,6 +180,8 @@ class Item(BaseModel): models: List[Model] callback_url: str model_volume_name: str + run_timeout: Optional[int] = Field(default=60 * 5) + idle_timeout: Optional[int] = Field(default=60) gpu: GPUType = Field(default=GPUType.T4) @field_validator('gpu') @@ -391,7 +393,9 @@ async def build_logic(item: Item): "gpu": item.gpu, "public_model_volume": public_model_volume_name, "private_model_volume": item.model_volume_name, - "pip": list(pip_modules) + "pip": list(pip_modules), + "run_timeout": item.run_timeout, + "idle_timeout": item.idle_timeout, } with open(f"{folder_path}/config.py", "w") as f: f.write("config = " + json.dumps(config)) diff --git a/builder/modal-builder/src/template/app.py b/builder/modal-builder/src/template/app.py index 2131014..8ab86c2 100644 --- a/builder/modal-builder/src/template/app.py +++ b/builder/modal-builder/src/template/app.py @@ -112,7 +112,7 @@ def check_server(url, retries=50, delay=500): # If the response status code is 200, the server is up and running if response.status_code == 200: - print(f"runpod-worker-comfy - API is reachable") + print(f"comfy-modal - API is reachable") return True except requests.RequestException as e: # If an exception occurs, the server may not be ready @@ -124,7 +124,7 @@ def check_server(url, retries=50, delay=500): time.sleep(delay / 1000) print( - f"runpod-worker-comfy - Failed to connect to server at {url} after {retries} attempts." + f"comfy-modal - Failed to connect to server at {url} after {retries} attempts." ) return False @@ -158,20 +158,68 @@ image = Image.debian_slim() target_image = image if deploy_test else dockerfile_image -@stub.cls(image=target_image, gpu=config["gpu"] ,volumes=volumes, timeout=60 * 10, container_idle_timeout=60) +run_timeout = config["run_timeout"] +idle_timeout = config["idle_timeout"] + +import asyncio + +@stub.cls(image=target_image, gpu=config["gpu"] ,volumes=volumes, timeout=60 * 10, container_idle_timeout=idle_timeout) class ComfyDeployRunner: + machine_logs = [] + + async def read_stream(self, stream, isStderr): + import time + while True: + line = await stream.readline() + if line: + l = line.decode('utf-8').strip() + + if l == "": + continue + + if not isStderr: + print(l, flush=True) + self.machine_logs.append({ + "logs": l, + "timestamp": time.time() + }) + + else: + # is error + # logger.error(l) + print(l, flush=True) + self.machine_logs.append({ + "logs": l, + "timestamp": time.time() + }) + else: + break + @enter() - def setup(self): + async def setup(self): import subprocess import time # Make sure that the ComfyUI API is available print(f"comfy-modal - check server") - command = ["python", "main.py", - "--disable-auto-launch", "--disable-metadata"] + # command = ["python", "main.py", + # "--disable-auto-launch", "--disable-metadata"] - self.server_process = subprocess.Popen(command, cwd="/comfyui") + # self.server_process = subprocess.Popen(command, cwd="/comfyui") + + self.server_process = await asyncio.subprocess.create_subprocess_shell( + f"python main.py --disable-auto-launch --disable-metadata", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd="/comfyui", + # env={**os.environ, "COLUMNS": "10000"} + ) + + self.stdout_task = asyncio.create_task( + self.read_stream(self.server_process.stdout, False)) + self.stderr_task = asyncio.create_task( + self.read_stream(self.server_process.stderr, True)) check_server( f"http://{COMFY_HOST}", @@ -180,63 +228,108 @@ class ComfyDeployRunner: ) @exit() - def cleanup(self, exc_type, exc_value, traceback): - self.server_process.terminate() + async def cleanup(self, exc_type, exc_value, traceback): + print(f"comfy-modal - cleanup", exc_type, exc_value, traceback) + self.stderr_task.cancel() + self.stdout_task.cancel() + # self.server_process.kill() @method() - def run(self, input: Input): + async def run(self, input: Input): + import signal + import time + # import asyncio + + self.stderr_task.cancel() + self.stdout_task.cancel() + + self.stdout_task = asyncio.create_task( + self.read_stream(self.server_process.stdout, False)) + self.stderr_task = asyncio.create_task( + self.read_stream(self.server_process.stderr, True)) + + class TimeoutError(Exception): + pass + + def timeout_handler(signum, frame): + data = json.dumps({ + "run_id": input.prompt_id, + "status": "timeout", + "time": datetime.now().isoformat() + }).encode('utf-8') + req = urllib.request.Request(input.status_endpoint, data=data, method='POST') + urllib.request.urlopen(req) + raise TimeoutError("Operation timed out") + + signal.signal(signal.SIGALRM, timeout_handler) + + try: + # Set an alarm for some seconds in the future + signal.alarm(run_timeout) # 5 seconds timeout + + data = json.dumps({ + "run_id": input.prompt_id, + "status": "started", + "time": datetime.now().isoformat() + }).encode('utf-8') + req = urllib.request.Request(input.status_endpoint, data=data, method='POST') + urllib.request.urlopen(req) + + job_input = input + + try: + queued_workflow = queue_workflow_comfy_deploy(job_input) # queue_workflow(workflow) + prompt_id = queued_workflow["prompt_id"] + print(f"comfy-modal - queued workflow with ID {prompt_id}") + except Exception as e: + import traceback + print(traceback.format_exc()) + return {"error": f"Error queuing workflow: {str(e)}"} + + # Poll for completion + print(f"comfy-modal - wait until image generation is complete") + retries = 0 + status = "" + try: + print("getting request") + while retries < COMFY_POLLING_MAX_RETRIES: + status_result = check_status(prompt_id=prompt_id) + if 'status' in status_result and (status_result['status'] == 'success' or status_result['status'] == 'failed'): + status = status_result['status'] + print(status) + break + else: + # Wait before trying again + time.sleep(COMFY_POLLING_INTERVAL_MS / 1000) + retries += 1 + else: + return {"error": "Max retries reached while waiting for image generation"} + except Exception as e: + return {"error": f"Error waiting for image generation: {str(e)}"} + + print(f"comfy-modal - Finished, turning off") + + result = {"status": status} + + except TimeoutError: + print("Operation timed out") + return {"status": "failed"} + + + print("uploading log_data") data = json.dumps({ "run_id": input.prompt_id, - "status": "started", - "time": datetime.now().isoformat() + "time": datetime.now().isoformat(), + "log_data": json.dumps(self.machine_logs) }).encode('utf-8') + print("my logs", len(self.machine_logs)) + # Clear logs + self.machine_logs = [] req = urllib.request.Request(input.status_endpoint, data=data, method='POST') urllib.request.urlopen(req) - - job_input = input - - try: - queued_workflow = queue_workflow_comfy_deploy(job_input) # queue_workflow(workflow) - prompt_id = queued_workflow["prompt_id"] - print(f"comfy-modal - queued workflow with ID {prompt_id}") - except Exception as e: - import traceback - print(traceback.format_exc()) - return {"error": f"Error queuing workflow: {str(e)}"} - - # Poll for completion - print(f"comfy-modal - wait until image generation is complete") - retries = 0 - status = "" - try: - print("getting request") - while retries < COMFY_POLLING_MAX_RETRIES: - status_result = check_status(prompt_id=prompt_id) - # history = get_history(prompt_id) - - # Exit the loop if we have found the history - # if prompt_id in history and history[prompt_id].get("outputs"): - # break - - # Exit the loop if we have found the status both success or failed - if 'status' in status_result and (status_result['status'] == 'success' or status_result['status'] == 'failed'): - status = status_result['status'] - print(status) - break - else: - # Wait before trying again - time.sleep(COMFY_POLLING_INTERVAL_MS / 1000) - retries += 1 - else: - return {"error": "Max retries reached while waiting for image generation"} - except Exception as e: - return {"error": f"Error waiting for image generation: {str(e)}"} - - print(f"comfy-modal - Finished, turning off") - - result = {"status": status} - + return result + @web_app.post("/run") @@ -252,10 +345,12 @@ async def post_run(request_input: RequestInput): urllib.request.urlopen(req) model = ComfyDeployRunner() - call = model.run.spawn(request_input.input) + call = await model.run.spawn.aio(request_input.input) + + print("call", call) # call = run.spawn() - return {"call_id": call.object_id} + return {"call_id": None} return {"call_id": None} diff --git a/builder/modal-builder/src/template/config.py b/builder/modal-builder/src/template/config.py index 62c21a2..dd2f92d 100644 --- a/builder/modal-builder/src/template/config.py +++ b/builder/modal-builder/src/template/config.py @@ -4,5 +4,7 @@ config = { "gpu": "T4", "public_model_volume": "model-store", "private_model_volume": "private-model-store", - "pip": [] + "pip": [], + "run_timeout": 60 * 5, + "idle_timeout": 60 } diff --git a/web/drizzle/0048_dear_korath.sql b/web/drizzle/0048_dear_korath.sql new file mode 100644 index 0000000..eec2493 --- /dev/null +++ b/web/drizzle/0048_dear_korath.sql @@ -0,0 +1,2 @@ +ALTER TYPE "workflow_run_status" ADD VALUE 'timeout';--> statement-breakpoint +ALTER TABLE "comfyui_deploy"."workflow_runs" ADD COLUMN "run_log" text; \ No newline at end of file diff --git a/web/drizzle/meta/0048_snapshot.json b/web/drizzle/meta/0048_snapshot.json new file mode 100644 index 0000000..82c7d20 --- /dev/null +++ b/web/drizzle/meta/0048_snapshot.json @@ -0,0 +1,1305 @@ +{ + "id": "b5ef1a74-3b9d-420e-8644-8d7d304b00ab", + "prevId": "42e70101-5904-4148-abcf-a655b3341bdd", + "version": "5", + "dialect": "pg", + "tables": { + "api_keys": { + "name": "api_keys", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_users_id_fk": { + "name": "api_keys_user_id_users_id_fk", + "tableFrom": "api_keys", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_key_unique": { + "name": "api_keys_key_unique", + "nullsNotDistinct": false, + "columns": [ + "key" + ] + } + } + }, + "auth_requests": { + "name": "auth_requests", + "schema": "comfyui_deploy", + "columns": { + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_hash": { + "name": "api_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expired_date": { + "name": "expired_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "deployments": { + "name": "deployments", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_version_id": { + "name": "workflow_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "machine_id": { + "name": "machine_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "share_slug": { + "name": "share_slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "showcase_media": { + "name": "showcase_media", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "environment": { + "name": "environment", + "type": "deployment_environment", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "deployments_user_id_users_id_fk": { + "name": "deployments_user_id_users_id_fk", + "tableFrom": "deployments", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deployments_workflow_version_id_workflow_versions_id_fk": { + "name": "deployments_workflow_version_id_workflow_versions_id_fk", + "tableFrom": "deployments", + "tableTo": "workflow_versions", + "columnsFrom": [ + "workflow_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployments_workflow_id_workflows_id_fk": { + "name": "deployments_workflow_id_workflows_id_fk", + "tableFrom": "deployments", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deployments_machine_id_machines_id_fk": { + "name": "deployments_machine_id_machines_id_fk", + "tableFrom": "deployments", + "tableTo": "machines", + "columnsFrom": [ + "machine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployments_share_slug_unique": { + "name": "deployments_share_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "share_slug" + ] + } + } + }, + "machines": { + "name": "machines", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auth_token": { + "name": "auth_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "machine_type", + "primaryKey": false, + "notNull": true, + "default": "'classic'" + }, + "status": { + "name": "status", + "type": "machine_status", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "models": { + "name": "models", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "gpu": { + "name": "gpu", + "type": "machine_gpu", + "primaryKey": false, + "notNull": false + }, + "build_machine_instance_id": { + "name": "build_machine_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "build_log": { + "name": "build_log", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "machines_user_id_users_id_fk": { + "name": "machines_user_id_users_id_fk", + "tableFrom": "machines", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "models": { + "name": "models", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_volume_id": { + "name": "user_volume_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_path": { + "name": "folder_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_id": { + "name": "civitai_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_version_id": { + "name": "civitai_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_url": { + "name": "civitai_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_download_url": { + "name": "civitai_download_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_model_response": { + "name": "civitai_model_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hf_url": { + "name": "hf_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_url": { + "name": "s3_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_url": { + "name": "client_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "status": { + "name": "status", + "type": "resource_upload", + "primaryKey": false, + "notNull": true, + "default": "'started'" + }, + "upload_machine_id": { + "name": "upload_machine_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upload_type": { + "name": "upload_type", + "type": "model_upload_type", + "primaryKey": false, + "notNull": true + }, + "model_type": { + "name": "model_type", + "type": "model_type", + "primaryKey": false, + "notNull": true, + "default": "'checkpoint'" + }, + "error_log": { + "name": "error_log", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "models_user_id_users_id_fk": { + "name": "models_user_id_users_id_fk", + "tableFrom": "models", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "models_user_volume_id_user_volume_id_fk": { + "name": "models_user_volume_id_user_volume_id_fk", + "tableFrom": "models", + "tableTo": "user_volume", + "columnsFrom": [ + "user_volume_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "subscription_status": { + "name": "subscription_status", + "schema": "comfyui_deploy", + "columns": { + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan": { + "name": "plan", + "type": "subscription_plan", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "subscription_plan_status", + "primaryKey": false, + "notNull": true + }, + "subscription_id": { + "name": "subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscription_item_plan_id": { + "name": "subscription_item_plan_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscription_item_api_id": { + "name": "subscription_item_api_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cancel_at_period_end": { + "name": "cancel_at_period_end", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "user_usage": { + "name": "user_usage", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "usage_time": { + "name": "usage_time", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_usage_user_id_users_id_fk": { + "name": "user_usage_user_id_users_id_fk", + "tableFrom": "user_usage", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "user_volume": { + "name": "user_volume", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "volume_name": { + "name": "volume_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": { + "user_volume_user_id_users_id_fk": { + "name": "user_volume_user_id_users_id_fk", + "tableFrom": "user_volume", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "users": { + "name": "users", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_run_outputs": { + "name": "workflow_run_outputs", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_run_outputs_run_id_workflow_runs_id_fk": { + "name": "workflow_run_outputs_run_id_workflow_runs_id_fk", + "tableFrom": "workflow_run_outputs", + "tableTo": "workflow_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_runs": { + "name": "workflow_runs", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workflow_version_id": { + "name": "workflow_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "workflow_inputs": { + "name": "workflow_inputs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "machine_id": { + "name": "machine_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "workflow_run_origin", + "primaryKey": false, + "notNull": true, + "default": "'api'" + }, + "status": { + "name": "status", + "type": "workflow_run_status", + "primaryKey": false, + "notNull": true, + "default": "'not-started'" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "gpu": { + "name": "gpu", + "type": "machine_gpu", + "primaryKey": false, + "notNull": false + }, + "machine_type": { + "name": "machine_type", + "type": "machine_type", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "run_log": { + "name": "run_log", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_runs_workflow_version_id_workflow_versions_id_fk": { + "name": "workflow_runs_workflow_version_id_workflow_versions_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "workflow_versions", + "columnsFrom": [ + "workflow_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_runs_workflow_id_workflows_id_fk": { + "name": "workflow_runs_workflow_id_workflows_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_runs_machine_id_machines_id_fk": { + "name": "workflow_runs_machine_id_machines_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "machines", + "columnsFrom": [ + "machine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflows": { + "name": "workflows", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflows_user_id_users_id_fk": { + "name": "workflows_user_id_users_id_fk", + "tableFrom": "workflows", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_versions": { + "name": "workflow_versions", + "schema": "comfyui_deploy", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workflow": { + "name": "workflow", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workflow_api": { + "name": "workflow_api", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_versions_workflow_id_workflows_id_fk": { + "name": "workflow_versions_workflow_id_workflows_id_fk", + "tableFrom": "workflow_versions", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "deployment_environment": { + "name": "deployment_environment", + "values": { + "staging": "staging", + "production": "production", + "public-share": "public-share" + } + }, + "machine_gpu": { + "name": "machine_gpu", + "values": { + "T4": "T4", + "A10G": "A10G", + "A100": "A100" + } + }, + "machine_status": { + "name": "machine_status", + "values": { + "ready": "ready", + "building": "building", + "error": "error" + } + }, + "machine_type": { + "name": "machine_type", + "values": { + "classic": "classic", + "runpod-serverless": "runpod-serverless", + "modal-serverless": "modal-serverless", + "comfy-deploy-serverless": "comfy-deploy-serverless" + } + }, + "model_type": { + "name": "model_type", + "values": { + "checkpoint": "checkpoint", + "lora": "lora", + "embedding": "embedding", + "vae": "vae" + } + }, + "model_upload_type": { + "name": "model_upload_type", + "values": { + "civitai": "civitai", + "download-url": "download-url", + "huggingface": "huggingface", + "other": "other" + } + }, + "resource_upload": { + "name": "resource_upload", + "values": { + "started": "started", + "success": "success", + "failed": "failed" + } + }, + "subscription_plan": { + "name": "subscription_plan", + "values": { + "basic": "basic", + "pro": "pro", + "enterprise": "enterprise" + } + }, + "subscription_plan_status": { + "name": "subscription_plan_status", + "values": { + "active": "active", + "deleted": "deleted", + "paused": "paused" + } + }, + "workflow_run_origin": { + "name": "workflow_run_origin", + "values": { + "manual": "manual", + "api": "api", + "public-share": "public-share" + } + }, + "workflow_run_status": { + "name": "workflow_run_status", + "values": { + "not-started": "not-started", + "running": "running", + "uploading": "uploading", + "success": "success", + "failed": "failed", + "started": "started", + "queued": "queued", + "timeout": "timeout" + } + } + }, + "schemas": { + "comfyui_deploy": "comfyui_deploy" + }, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/web/drizzle/meta/_journal.json b/web/drizzle/meta/_journal.json index 8220257..3524c40 100644 --- a/web/drizzle/meta/_journal.json +++ b/web/drizzle/meta/_journal.json @@ -337,6 +337,13 @@ "when": 1706384528895, "tag": "0047_gifted_starbolt", "breakpoints": true + }, + { + "idx": 48, + "version": "5", + "when": 1706600255919, + "tag": "0048_dear_korath", + "breakpoints": true } ] } \ No newline at end of file diff --git a/web/src/app/(app)/api/update-run/route.ts b/web/src/app/(app)/api/update-run/route.ts index c80f0ae..012fa6a 100644 --- a/web/src/app/(app)/api/update-run/route.ts +++ b/web/src/app/(app)/api/update-run/route.ts @@ -2,10 +2,8 @@ import { parseDataSafe } from "../../../../lib/parseDataSafe"; import { db } from "@/db/db"; import { WorkflowRunStatusSchema, - userUsageTable, workflowRunOutputs, workflowRunsTable, - workflowTable, } from "@/db/schema"; import { getCurrentPlan } from "@/server/getCurrentPlan"; import { stripe } from "@/server/stripe"; @@ -18,6 +16,7 @@ const Request = z.object({ status: WorkflowRunStatusSchema.optional(), time: z.coerce.date().optional(), output_data: z.any().optional(), + log_data: z.string().optional(), }); export async function POST(request: Request) { @@ -26,7 +25,17 @@ export async function POST(request: Request) { if (!data || error) return error; - const { run_id, status, time, output_data } = data; + const { run_id, status, time, output_data, log_data } = data; + + if (log_data) { + // It successfully started, update the started_at time + await db + .update(workflowRunsTable) + .set({ + run_log: log_data, + }) + .where(eq(workflowRunsTable.id, run_id)); + } if (status == "started" && time != undefined) { // It successfully started, update the started_at time @@ -48,6 +57,9 @@ export async function POST(request: Request) { .where(eq(workflowRunsTable.id, run_id)); } + const ended = + status === "success" || status === "failed" || status === "timeout"; + if (output_data) { const workflow_run_output = await db.insert(workflowRunOutputs).values({ run_id: run_id, @@ -58,8 +70,7 @@ export async function POST(request: Request) { .update(workflowRunsTable) .set({ status: status, - ended_at: - status === "success" || status === "failed" ? new Date() : null, + ended_at: ended ? new Date() : null, }) .where(eq(workflowRunsTable.id, run_id)) .returning(); @@ -67,10 +78,7 @@ export async function POST(request: Request) { // Need to filter out only comfy deploy serverless // Also multiply with the gpu selection if (workflow_run.machine_type == "comfy-deploy-serverless") { - if ( - (status === "success" || status === "failed") && - workflow_run.user_id - ) { + if (ended && workflow_run.user_id) { const sub = await getCurrentPlan({ user_id: workflow_run.user_id, org_id: workflow_run.org_id, diff --git a/web/src/components/LiveStatus.tsx b/web/src/components/LiveStatus.tsx index 867cf5f..4c6d0f6 100644 --- a/web/src/components/LiveStatus.tsx +++ b/web/src/components/LiveStatus.tsx @@ -16,7 +16,7 @@ export function LiveStatus({ (state) => state.data .filter((x) => x.id === run.id) - .sort((a, b) => b.timestamp - a.timestamp)?.[0] + .sort((a, b) => b.timestamp - a.timestamp)?.[0], ); let status = run.status; @@ -51,7 +51,9 @@ export function LiveStatus({ <> {data && status != "success" - ? `${data.json.event} - ${data.json.data.node}` + ? `${data.json.event}${ + data.json.data.node ? " - " + data.json.data.node : "" + }` : "-"} diff --git a/web/src/components/RunDisplay.tsx b/web/src/components/RunDisplay.tsx index 3d980ca..01e2379 100644 --- a/web/src/components/RunDisplay.tsx +++ b/web/src/components/RunDisplay.tsx @@ -19,6 +19,7 @@ import { getDuration, getRelativeTime } from "@/lib/getRelativeTime"; import { type findAllRuns } from "@/server/findAllRuns"; import { Suspense } from "react"; import { LiveStatus } from "./LiveStatus"; +import { LogsType, LogsViewer } from "@/components/LogsViewer"; export async function RunDisplay({ run, @@ -75,6 +76,9 @@ export async function RunDisplay({ + {run.run_log && ( + + )} {/*
{view}
*/} diff --git a/web/src/components/StatusBadge.tsx b/web/src/components/StatusBadge.tsx index b136b9d..87a793c 100644 --- a/web/src/components/StatusBadge.tsx +++ b/web/src/components/StatusBadge.tsx @@ -17,6 +17,8 @@ export function StatusBadge({ ); case "success": return {status}; + case "timeout": + return {status}; case "failed": return {status}; } diff --git a/web/src/db/schema.ts b/web/src/db/schema.ts index 8e30e8f..426e776 100644 --- a/web/src/db/schema.ts +++ b/web/src/db/schema.ts @@ -104,6 +104,7 @@ export const workflowRunStatus = pgEnum("workflow_run_status", [ "failed", "started", "queued", + "timeout", ]); export const deploymentEnvironment = pgEnum("deployment_environment", [ @@ -172,6 +173,7 @@ export const workflowRunsTable = dbSchema.table("workflow_runs", { machine_type: machinesType("machine_type"), user_id: text("user_id"), org_id: text("org_id"), + run_log: text("run_log"), }); export const workflowRunRelations = relations( @@ -385,13 +387,8 @@ export const modelUploadType = pgEnum("model_upload_type", [ "other", ]); -// https://www.answeroverflow.com/m/1125106227387584552 -export const modelTypes = [ - "checkpoint", - "lora", - "embedding", - "vae", -] as const +// https://www.answeroverflow.com/m/1125106227387584552 +export const modelTypes = ["checkpoint", "lora", "embedding", "vae"] as const; export const modelType = pgEnum("model_type", modelTypes); export type modelEnumType = (typeof modelTypes)[number];