diff --git a/builder/modal-builder/src/main.py b/builder/modal-builder/src/main.py index 2493a37..46787de 100644 --- a/builder/modal-builder/src/main.py +++ b/builder/modal-builder/src/main.py @@ -8,6 +8,7 @@ from enum import Enum import json import subprocess import time +from uuid import uuid4 from contextlib import asynccontextmanager import asyncio import threading @@ -19,6 +20,7 @@ from urllib.parse import parse_qs from starlette.middleware.base import BaseHTTPMiddleware from starlette.types import ASGIApp, Scope, Receive, Send + from concurrent.futures import ThreadPoolExecutor # executor = ThreadPoolExecutor(max_workers=5) @@ -45,6 +47,7 @@ machine_id_websocket_dict = {} machine_id_status = {} fly_instance_id = os.environ.get('FLY_ALLOC_ID', 'local').split('-')[0] +civitai_api_key = os.environ.get('FLY_ALLOC_ID', 'local').split('-')[0] class FlyReplayMiddleware(BaseHTTPMiddleware): @@ -174,6 +177,7 @@ class Item(BaseModel): snapshot: Snapshot models: List[Model] callback_url: str + checkpoint_volume_name: str gpu: GPUType = Field(default=GPUType.T4) @field_validator('gpu') @@ -223,6 +227,103 @@ async def websocket_endpoint(websocket: WebSocket, machine_id: str): # return {"Hello": "World"} +class UploadType(str, Enum): + checkpoint = "checkpoint" + +class UploadBody(BaseModel): + download_url: str + volume_name: str + volume_id: str + checkpoint_id: str + upload_type: UploadType + callback_url: str + + +UPLOAD_TYPE_DIR_MAP = { + UploadType.checkpoint: "checkpoints" +} + +@app.post("/upload-volume") +async def upload_checkpoint(body: UploadBody): + global last_activity_time + last_activity_time = time.time() + logger.info(f"Extended inactivity time to {global_timeout}") + + asyncio.create_task(upload_logic(body)) + + # check that this + return JSONResponse(status_code=200, content={"message": "Volume uploading", "build_machine_instance_id": fly_instance_id}) + +async def upload_logic(body: UploadBody): + folder_path = f"/app/builds/{body.volume_id}" + + cp_process = await asyncio.subprocess.create_subprocess_exec("cp", "-r", "/app/src/volume-builder", folder_path) + await cp_process.wait() + + upload_path = UPLOAD_TYPE_DIR_MAP[body.upload_type] + config = { + "volume_names": { + body.volume_name: {"download_url": body.download_url, "folder_path": upload_path} + }, + "volume_paths": { + body.volume_name: f'/volumes/{uuid4()}' + }, + "callback_url": body.callback_url, + "callback_body": { + "checkpoint_id": body.checkpoint_id, + "volume_id": body.volume_id, + "folder_path": upload_path, + }, + "civitai_api_key": os.environ.get('CIVITAI_API_KEY') + } + with open(f"{folder_path}/config.py", "w") as f: + f.write("config = " + json.dumps(config)) + + process = await asyncio.subprocess.create_subprocess_shell( + f"modal run app.py", + # stdout=asyncio.subprocess.PIPE, + # stderr=asyncio.subprocess.PIPE, + cwd=folder_path, + env={**os.environ, "COLUMNS": "10000"} + ) + + # error_logs = [] + # async def read_stream(stream): + # while True: + # line = await stream.readline() + # if line: + # l = line.decode('utf-8').strip() + # error_logs.append(l) + # logger.error(l) + # error_logs.append({ + # "logs": l, + # "timestamp": time.time() + # }) + # else: + # break + + # stderr_read_task = asyncio.create_task(read_stream(process.stderr)) + # + # await asyncio.wait([stderr_read_task]) + # await process.wait() + + # if process.returncode != 0: + # error_logs.append({"logs": "Unable to upload volume.", "timestamp": time.time()}) + # # Error handling: send POST request to callback URL with error details + # requests.post(body.callback_url, json={ + # "volume_id": body.volume_id, + # "checkpoint_id": body.checkpoint_id, + # "folder_path": upload_path, + # "error_logs": json.dumps(error_logs), + # "status": "failed" + # }) + # + # requests.post(body.callback_url, json={ + # "checkpoint_id": body.checkpoint_id, + # "volume_id": body.volume_id, + # "folder_path": upload_path, + # "status": "success" + # }) @app.post("/create") async def create_machine(item: Item): @@ -312,7 +413,9 @@ async def build_logic(item: Item): config = { "name": item.name, "deploy_test": os.environ.get("DEPLOY_TEST_FLAG", "False"), - "gpu": item.gpu + "gpu": item.gpu, + "public_checkpoint_volume": "model-store", + "private_checkpoint_volume": item.checkpoint_volume_name } with open(f"{folder_path}/config.py", "w") as f: f.write("config = " + json.dumps(config)) diff --git a/builder/modal-builder/src/template/app.py b/builder/modal-builder/src/template/app.py index 1d4145d..91d5601 100644 --- a/builder/modal-builder/src/template/app.py +++ b/builder/modal-builder/src/template/app.py @@ -1,12 +1,13 @@ from config import config import modal -from modal import Image, Mount, web_endpoint, Stub, asgi_app +from modal import Image, Mount, web_endpoint, Stub, asgi_app import json import urllib.request import urllib.parse from pydantic import BaseModel from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse +from volume_setup import volumes # deploy_test = False @@ -27,8 +28,8 @@ deploy_test = config["deploy_test"] == "True" web_app = FastAPI() print(config) print("deploy_test ", deploy_test) +print('volumes', volumes) stub = Stub(name=config["name"]) -# print(stub.app_id) if not deploy_test: # dockerfile_image = Image.from_dockerfile(f"{current_directory}/Dockerfile", context_mount=Mount.from_local_dir(f"{current_directory}/data", remote_path="/data")) @@ -52,11 +53,13 @@ if not deploy_test: "cd /comfyui/custom_nodes/ComfyUI-Manager && pip install -r requirements.txt", "cd /comfyui/custom_nodes/ComfyUI-Manager && mkdir startup-scripts", ) + .run_commands(f"cat /comfyui/server.py") + .run_commands(f"ls /comfyui/app") # .run_commands( # # Install comfy deploy # "cd /comfyui/custom_nodes && git clone https://github.com/BennyKok/comfyui-deploy.git", # ) - # .copy_local_file(f"{current_directory}/data/extra_model_paths.yaml", "/comfyui") + .copy_local_file(f"{current_directory}/data/extra_model_paths.yaml", "/comfyui") .copy_local_file(f"{current_directory}/data/start.sh", "/start.sh") .run_commands("chmod +x /start.sh") @@ -153,8 +156,9 @@ image = Image.debian_slim() target_image = image if deploy_test else dockerfile_image - -@stub.function(image=target_image, gpu=config["gpu"]) +@stub.function(image=target_image, gpu=config["gpu"] + ,volumes=volumes +) def run(input: Input): import subprocess import time @@ -163,6 +167,7 @@ def run(input: Input): command = ["python", "main.py", "--disable-auto-launch", "--disable-metadata"] + server_process = subprocess.Popen(command, cwd="/comfyui") check_server( @@ -235,7 +240,9 @@ async def bar(request_input: RequestInput): # pass -@stub.function(image=image) +@stub.function(image=image + ,volumes=volumes +) @asgi_app() def comfyui_api(): return web_app @@ -285,6 +292,7 @@ def spawn_comfyui_in_background(): # to be on a single container. concurrency_limit=1, timeout=10 * 60, + volumes=volumes, ) @asgi_app() def comfyui_app(): @@ -303,4 +311,4 @@ def comfyui_app(): }, )() - return make_simple_proxy_app(ProxyContext(config)) \ No newline at end of file + return make_simple_proxy_app(ProxyContext(config)) diff --git a/builder/modal-builder/src/template/config.py b/builder/modal-builder/src/template/config.py index e59020b..a651642 100644 --- a/builder/modal-builder/src/template/config.py +++ b/builder/modal-builder/src/template/config.py @@ -1 +1,7 @@ -config = {"name": "my-app", "deploy_test": "True", "gpu": "T4"} \ No newline at end of file +config = { + "name": "my-app", + "deploy_test": "True", + "gpu": "T4", + "public_checkpoint_volume": "model-store", + "private_checkpoint_volume": "private-model-store" +} diff --git a/builder/modal-builder/src/template/data/extra_model_paths.yaml b/builder/modal-builder/src/template/data/extra_model_paths.yaml index 6e07d7b..66f758e 100644 --- a/builder/modal-builder/src/template/data/extra_model_paths.yaml +++ b/builder/modal-builder/src/template/data/extra_model_paths.yaml @@ -1,11 +1,15 @@ -comfyui: - base_path: /runpod-volume/ComfyUI/ - checkpoints: models/checkpoints/ - clip: models/clip/ - clip_vision: models/clip_vision/ - configs: models/configs/ - controlnet: models/controlnet/ - embeddings: models/embeddings/ - loras: models/loras/ - upscale_models: models/upscale_models/ - vae: models/vae/ \ No newline at end of file +public: + base_path: /public_models/ + checkpoints: checkpoints + clip: clip + clip_vision: clip_vision + configs: configs + controlnet: controlnet + embeddings: embeddings + loras: loras + upscale_models: upscale_models + vae: vae + +private: + base_path: /private_models/ + checkpoints: checkpoints diff --git a/builder/modal-builder/src/template/data/install_deps.py b/builder/modal-builder/src/template/data/install_deps.py index 3ff3ca3..22a184e 100644 --- a/builder/modal-builder/src/template/data/install_deps.py +++ b/builder/modal-builder/src/template/data/install_deps.py @@ -54,4 +54,4 @@ for model in models: # Close the server server_process.terminate() -print("Finished installing dependencies.") \ No newline at end of file +print("Finished installing dependencies.") diff --git a/builder/modal-builder/src/template/volume_setup.py b/builder/modal-builder/src/template/volume_setup.py new file mode 100644 index 0000000..fb50e9f --- /dev/null +++ b/builder/modal-builder/src/template/volume_setup.py @@ -0,0 +1,9 @@ +import modal +from config import config + +public_model_volume = modal.Volume.persisted(config["public_checkpoint_volume"]) +private_volume = modal.Volume.persisted(config["private_checkpoint_volume"]) + +PUBLIC_BASEMODEL_DIR = "/public_models" +PRIVATE_BASEMODEL_DIR = "/private_models" +volumes = {PUBLIC_BASEMODEL_DIR: public_model_volume, PRIVATE_BASEMODEL_DIR: private_volume} diff --git a/builder/modal-builder/src/volume-builder/app.py b/builder/modal-builder/src/volume-builder/app.py new file mode 100644 index 0000000..9f5bd73 --- /dev/null +++ b/builder/modal-builder/src/volume-builder/app.py @@ -0,0 +1,74 @@ +import modal +from config import config +import os +import subprocess +from pprint import pprint + +stub = modal.Stub() + +# Volume names may only contain alphanumeric characters, dashes, periods, and underscores, and must be less than 64 characters in length. +def is_valid_name(name: str) -> bool: + allowed_characters = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._") + return 0 < len(name) <= 64 and all(char in allowed_characters for char in name) + +def create_volumes(volume_names, paths): + path_to_vol = {} + for volume_name in volume_names.keys(): + if not is_valid_name(volume_name): + pass + modal_volume = modal.Volume.persisted(volume_name) + path_to_vol[paths[volume_name]] = modal_volume + + return path_to_vol + +vol_name_to_links = config["volume_names"] +vol_name_to_path = config["volume_paths"] +callback_url = config["callback_url"] +callback_body = config["callback_body"] +civitai_key = config["civitai_api_key"] + +volumes = create_volumes(vol_name_to_links, vol_name_to_path) +image = ( + modal.Image.debian_slim().apt_install("wget").pip_install("requests") +) + +# download config { "download_url": "", "folder_path": ""} +timeout=5000 +@stub.function(volumes=volumes, image=image, timeout=timeout, gpu=None) +def download_model(volume_name, download_config): + import requests + download_url = download_config["download_url"] + folder_path = download_config["folder_path"] + + volume_base_path = vol_name_to_path[volume_name] + model_store_path = os.path.join(volume_base_path, folder_path) + modified_download_url = download_url + ("&" if "?" in download_url else "?") + "token=" + civitai_key + print('downloading', modified_download_url) + + subprocess.run(["wget", modified_download_url , "--content-disposition", "-P", model_store_path]) + subprocess.run(["ls", "-la", volume_base_path]) + subprocess.run(["ls", "-la", model_store_path]) + volumes[volume_base_path].commit() + + + status = {"status": "success"} + requests.post(callback_url, json={**status, **callback_body}) + print(f"finished! sending to {callback_url}") + pprint({**status, **callback_body}) + +@stub.local_entrypoint() +def simple_download(): + import requests + try: + list(download_model.starmap([(vol_name, link) for vol_name,link in vol_name_to_links.items()])) + except modal.exception.FunctionTimeoutError as e: + status = {"status": "failed", "error_logs": f"{str(e)}", "timeout": timeout} + requests.post(callback_url, json={**status, **callback_body}) + print(f"finished! sending to {callback_url}") + pprint({**status, **callback_body}) + except Exception as e: + status = {"status": "failed", "error_logs": str(e)} + requests.post(callback_url, json={**status, **callback_body}) + print(f"finished! sending to {callback_url}") + pprint({**status, **callback_body}) + diff --git a/builder/modal-builder/src/volume-builder/config.py b/builder/modal-builder/src/volume-builder/config.py new file mode 100644 index 0000000..3abfeac --- /dev/null +++ b/builder/modal-builder/src/volume-builder/config.py @@ -0,0 +1,18 @@ +config = { + "volume_names": { + "test": { + "download_url": "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose (1).png", + "folder_path": "images" + } + }, + "volume_paths": { + "test": "/volumes/something" + }, + "callback_url": "", + "callback_body": { + "checkpoint_id": "", + "volume_id": "", + "folder_path": "images", + }, + "civitai_api_key": "", +} diff --git a/web/drizzle/0042_windy_madelyne_pryor.sql b/web/drizzle/0042_windy_madelyne_pryor.sql new file mode 100644 index 0000000..c600712 --- /dev/null +++ b/web/drizzle/0042_windy_madelyne_pryor.sql @@ -0,0 +1,64 @@ +DO $$ BEGIN + CREATE TYPE "model_upload_type" AS ENUM('civitai', 'huggingface', 'other'); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + CREATE TYPE "resource_upload" AS ENUM('started', 'success', 'failed'); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "comfyui_deploy"."checkpoints" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "user_id" text, + "org_id" text, + "description" text, + "checkpoint_volume_id" uuid NOT NULL, + "model_name" text, + "folder_path" text, + "civitai_id" text, + "civitai_version_id" text, + "civitai_url" text, + "civitai_download_url" text, + "civitai_model_response" jsonb, + "hf_url" text, + "s3_url" text, + "client_url" text, + "is_public" boolean DEFAULT false NOT NULL, + "status" "resource_upload" DEFAULT 'started' NOT NULL, + "upload_machine_id" text, + "upload_type" "model_upload_type" NOT NULL, + "error_log" text, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "comfyui_deploy"."checkpoint_volume" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "user_id" text, + "org_id" text, + "volume_name" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + "disabled" boolean DEFAULT false NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "comfyui_deploy"."checkpoints" ADD CONSTRAINT "checkpoints_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "comfyui_deploy"."users"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "comfyui_deploy"."checkpoints" ADD CONSTRAINT "checkpoints_checkpoint_volume_id_checkpoint_volume_id_fk" FOREIGN KEY ("checkpoint_volume_id") REFERENCES "comfyui_deploy"."checkpoint_volume"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "comfyui_deploy"."checkpoint_volume" ADD CONSTRAINT "checkpoint_volume_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "comfyui_deploy"."users"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; diff --git a/web/drizzle/meta/0042_snapshot.json b/web/drizzle/meta/0042_snapshot.json new file mode 100644 index 0000000..f517368 --- /dev/null +++ b/web/drizzle/meta/0042_snapshot.json @@ -0,0 +1,1273 @@ +{ + "id": "4bbd69a1-bb1f-467b-a6d8-8412142c4c32", + "prevId": "ca1ab9de-49df-4d4a-81ae-d3db96bf55ad", + "version": "5", + "dialect": "pg", + "tables": { + "api_keys": { + "name": "api_keys", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_users_id_fk": { + "name": "api_keys_user_id_users_id_fk", + "tableFrom": "api_keys", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_key_unique": { + "name": "api_keys_key_unique", + "nullsNotDistinct": false, + "columns": [ + "key" + ] + } + } + }, + "auth_requests": { + "name": "auth_requests", + "schema": "comfyui_deploy", + "columns": { + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_hash": { + "name": "api_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expired_date": { + "name": "expired_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "checkpoints": { + "name": "checkpoints", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checkpoint_volume_id": { + "name": "checkpoint_volume_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "folder_path": { + "name": "folder_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_id": { + "name": "civitai_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_version_id": { + "name": "civitai_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_url": { + "name": "civitai_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_download_url": { + "name": "civitai_download_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_model_response": { + "name": "civitai_model_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hf_url": { + "name": "hf_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_url": { + "name": "s3_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_url": { + "name": "client_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "resource_upload", + "primaryKey": false, + "notNull": true, + "default": "'started'" + }, + "upload_machine_id": { + "name": "upload_machine_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upload_type": { + "name": "upload_type", + "type": "model_upload_type", + "primaryKey": false, + "notNull": true + }, + "error_log": { + "name": "error_log", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "checkpoints_user_id_users_id_fk": { + "name": "checkpoints_user_id_users_id_fk", + "tableFrom": "checkpoints", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "checkpoints_checkpoint_volume_id_checkpoint_volume_id_fk": { + "name": "checkpoints_checkpoint_volume_id_checkpoint_volume_id_fk", + "tableFrom": "checkpoints", + "tableTo": "checkpoint_volume", + "columnsFrom": [ + "checkpoint_volume_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "checkpoint_volume": { + "name": "checkpoint_volume", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "volume_name": { + "name": "volume_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": { + "checkpoint_volume_user_id_users_id_fk": { + "name": "checkpoint_volume_user_id_users_id_fk", + "tableFrom": "checkpoint_volume", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "deployments": { + "name": "deployments", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_version_id": { + "name": "workflow_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "machine_id": { + "name": "machine_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "share_slug": { + "name": "share_slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "showcase_media": { + "name": "showcase_media", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "environment": { + "name": "environment", + "type": "deployment_environment", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "deployments_user_id_users_id_fk": { + "name": "deployments_user_id_users_id_fk", + "tableFrom": "deployments", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deployments_workflow_version_id_workflow_versions_id_fk": { + "name": "deployments_workflow_version_id_workflow_versions_id_fk", + "tableFrom": "deployments", + "tableTo": "workflow_versions", + "columnsFrom": [ + "workflow_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployments_workflow_id_workflows_id_fk": { + "name": "deployments_workflow_id_workflows_id_fk", + "tableFrom": "deployments", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deployments_machine_id_machines_id_fk": { + "name": "deployments_machine_id_machines_id_fk", + "tableFrom": "deployments", + "tableTo": "machines", + "columnsFrom": [ + "machine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployments_share_slug_unique": { + "name": "deployments_share_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "share_slug" + ] + } + } + }, + "machines": { + "name": "machines", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auth_token": { + "name": "auth_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "machine_type", + "primaryKey": false, + "notNull": true, + "default": "'classic'" + }, + "status": { + "name": "status", + "type": "machine_status", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "models": { + "name": "models", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "gpu": { + "name": "gpu", + "type": "machine_gpu", + "primaryKey": false, + "notNull": false + }, + "build_machine_instance_id": { + "name": "build_machine_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "build_log": { + "name": "build_log", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "machines_user_id_users_id_fk": { + "name": "machines_user_id_users_id_fk", + "tableFrom": "machines", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "subscription_status": { + "name": "subscription_status", + "schema": "comfyui_deploy", + "columns": { + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan": { + "name": "plan", + "type": "subscription_plan", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "subscription_plan_status", + "primaryKey": false, + "notNull": true + }, + "subscription_id": { + "name": "subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscription_item_plan_id": { + "name": "subscription_item_plan_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subscription_item_api_id": { + "name": "subscription_item_api_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cancel_at_period_end": { + "name": "cancel_at_period_end", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "user_usage": { + "name": "user_usage", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "usage_time": { + "name": "usage_time", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_usage_user_id_users_id_fk": { + "name": "user_usage_user_id_users_id_fk", + "tableFrom": "user_usage", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "users": { + "name": "users", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_run_outputs": { + "name": "workflow_run_outputs", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_run_outputs_run_id_workflow_runs_id_fk": { + "name": "workflow_run_outputs_run_id_workflow_runs_id_fk", + "tableFrom": "workflow_run_outputs", + "tableTo": "workflow_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_runs": { + "name": "workflow_runs", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workflow_version_id": { + "name": "workflow_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "workflow_inputs": { + "name": "workflow_inputs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "machine_id": { + "name": "machine_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "workflow_run_origin", + "primaryKey": false, + "notNull": true, + "default": "'api'" + }, + "status": { + "name": "status", + "type": "workflow_run_status", + "primaryKey": false, + "notNull": true, + "default": "'not-started'" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "gpu": { + "name": "gpu", + "type": "machine_gpu", + "primaryKey": false, + "notNull": false + }, + "machine_type": { + "name": "machine_type", + "type": "machine_type", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_runs_workflow_version_id_workflow_versions_id_fk": { + "name": "workflow_runs_workflow_version_id_workflow_versions_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "workflow_versions", + "columnsFrom": [ + "workflow_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_runs_workflow_id_workflows_id_fk": { + "name": "workflow_runs_workflow_id_workflows_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_runs_machine_id_machines_id_fk": { + "name": "workflow_runs_machine_id_machines_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "machines", + "columnsFrom": [ + "machine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflows": { + "name": "workflows", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflows_user_id_users_id_fk": { + "name": "workflows_user_id_users_id_fk", + "tableFrom": "workflows", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_versions": { + "name": "workflow_versions", + "schema": "comfyui_deploy", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workflow": { + "name": "workflow", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workflow_api": { + "name": "workflow_api", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_versions_workflow_id_workflows_id_fk": { + "name": "workflow_versions_workflow_id_workflows_id_fk", + "tableFrom": "workflow_versions", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "deployment_environment": { + "name": "deployment_environment", + "values": { + "staging": "staging", + "production": "production", + "public-share": "public-share" + } + }, + "machine_gpu": { + "name": "machine_gpu", + "values": { + "T4": "T4", + "A10G": "A10G", + "A100": "A100" + } + }, + "machine_status": { + "name": "machine_status", + "values": { + "ready": "ready", + "building": "building", + "error": "error" + } + }, + "machine_type": { + "name": "machine_type", + "values": { + "classic": "classic", + "runpod-serverless": "runpod-serverless", + "modal-serverless": "modal-serverless", + "comfy-deploy-serverless": "comfy-deploy-serverless" + } + }, + "model_upload_type": { + "name": "model_upload_type", + "values": { + "civitai": "civitai", + "huggingface": "huggingface", + "other": "other" + } + }, + "resource_upload": { + "name": "resource_upload", + "values": { + "started": "started", + "success": "success", + "failed": "failed" + } + }, + "subscription_plan": { + "name": "subscription_plan", + "values": { + "basic": "basic", + "pro": "pro", + "enterprise": "enterprise" + } + }, + "subscription_plan_status": { + "name": "subscription_plan_status", + "values": { + "active": "active", + "deleted": "deleted", + "paused": "paused" + } + }, + "workflow_run_origin": { + "name": "workflow_run_origin", + "values": { + "manual": "manual", + "api": "api", + "public-share": "public-share" + } + }, + "workflow_run_status": { + "name": "workflow_run_status", + "values": { + "not-started": "not-started", + "running": "running", + "uploading": "uploading", + "success": "success", + "failed": "failed" + } + } + }, + "schemas": { + "comfyui_deploy": "comfyui_deploy" + }, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/web/drizzle/meta/_journal.json b/web/drizzle/meta/_journal.json index 60056be..19546f0 100644 --- a/web/drizzle/meta/_journal.json +++ b/web/drizzle/meta/_journal.json @@ -295,6 +295,13 @@ "when": 1706111421524, "tag": "0041_thick_norrin_radd", "breakpoints": true + }, + { + "idx": 42, + "version": "5", + "when": 1706164614659, + "tag": "0042_windy_madelyne_pryor", + "breakpoints": true } ] } \ No newline at end of file diff --git a/web/src/app/(app)/api/volume-upload/route.ts b/web/src/app/(app)/api/volume-upload/route.ts new file mode 100644 index 0000000..b21741e --- /dev/null +++ b/web/src/app/(app)/api/volume-upload/route.ts @@ -0,0 +1,55 @@ +import { parseDataSafe } from "../../../../lib/parseDataSafe"; +import { db } from "@/db/db"; +import { checkpointTable, machinesTable } from "@/db/schema"; +import { eq } from "drizzle-orm"; +import { NextResponse } from "next/server"; +import { z } from "zod"; + +const Request = z.object({ + volume_id: z.string(), + checkpoint_id: z.string(), + folder_path: z.string().optional(), + status: z.enum(['success', 'failed']), + error_log: z.string().optional(), + timeout: z.number().optional(), +}); + +export async function POST(request: Request) { + const [data, error] = await parseDataSafe(Request, request); + if (!data || error) return error; + + const { checkpoint_id, error_log, status, folder_path } = data; + console.log( checkpoint_id, error_log, status, folder_path ) + + if (status === "success") { + await db + .update(checkpointTable) + .set({ + status: "success", + folder_path, + updated_at: new Date(), + // build_log: build_log, + }) + .where(eq(checkpointTable.id, checkpoint_id)); + } else { + await db + .update(checkpointTable) + .set({ + status: "failed", + error_log, + updated_at: new Date(), + // status: "error", + // build_log: build_log, + }) + .where(eq(checkpointTable.id, checkpoint_id)); + } + + return NextResponse.json( + { + message: "success", + }, + { + status: 200, + } + ); +} diff --git a/web/src/app/(app)/storage/loading.tsx b/web/src/app/(app)/storage/loading.tsx new file mode 100644 index 0000000..9ff4783 --- /dev/null +++ b/web/src/app/(app)/storage/loading.tsx @@ -0,0 +1,9 @@ +"use client"; + +import { LoadingPageWrapper } from "@/components/LoadingWrapper"; +import { usePathname } from "next/navigation"; + +export default function Loading() { + const pathName = usePathname(); + return ; +} diff --git a/web/src/app/(app)/storage/page.tsx b/web/src/app/(app)/storage/page.tsx new file mode 100644 index 0000000..c17cc9b --- /dev/null +++ b/web/src/app/(app)/storage/page.tsx @@ -0,0 +1,35 @@ +import { setInitialUserData } from "../../../lib/setInitialUserData"; +import { auth } from "@clerk/nextjs"; +import { clerkClient } from "@clerk/nextjs/server"; +import { CheckpointList } from "@/components/CheckpointList"; +import { getAllUserCheckpoints } from "@/server/getAllUserCheckpoints"; + +export default function Page() { + return ; +} + +async function CheckpointListServer() { + const { userId } = auth(); + + if (!userId) { + return
No auth
; + } + + const user = await clerkClient.users.getUser(userId); + + if (!user) { + await setInitialUserData(userId); + } + + const checkpoints = await getAllUserCheckpoints(); + + if (!checkpoints) { + return
No checkpoints found
; + } + + return ( +
+ +
+ ); +} diff --git a/web/src/components/CheckpointList.tsx b/web/src/components/CheckpointList.tsx new file mode 100644 index 0000000..a0fce75 --- /dev/null +++ b/web/src/components/CheckpointList.tsx @@ -0,0 +1,373 @@ +"use client"; + +import { getRelativeTime } from "../lib/getRelativeTime"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { InsertModal, UpdateModal } from "./InsertModal"; +import { Input } from "@/components/ui/input"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import type { getAllUserCheckpoints } from "@/server/getAllUserCheckpoints"; +import type { + ColumnDef, + ColumnFiltersState, + SortingState, + VisibilityState, +} from "@tanstack/react-table"; +import { + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { ArrowUpDown } from "lucide-react"; +import * as React from "react"; +import { addCivitaiCheckpoint } from "@/server/curdCheckpoint"; +import { addCivitaiCheckpointSchema } from "@/server/addCheckpointSchema"; + +export type CheckpointItemList = NonNullable< + Awaited> +>[0]; + +export const columns: ColumnDef[] = [ + { + accessorKey: "id", + id: "select", + header: ({ table }) => ( + table.toggleAllPageRowsSelected(!!value)} + aria-label="Select all" + /> + ), + cell: ({ row }) => ( + row.toggleSelected(!!value)} + aria-label="Select row" + /> + ), + enableSorting: false, + enableHiding: false, + }, + { + accessorKey: "model_name", + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + const checkpoint = row.original; + return ( + + + {row.original.model_name} + + + {checkpoint.is_public ? ( + Public + ) : ( + Private + )} + + ); + }, + }, + { + accessorKey: "status", + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + return ( + + {row.original.status} + + ); + // NOTE: retry downloads on failures + // const oneHourAgo = new Date(new Date().getTime() - (60 * 60 * 1000)); + // const lastUpdated = new Date(row.original.updated_at); + // const canRefresh = row.original.status === "failed" && lastUpdated < oneHourAgo; + // const canRefresh = row.original.status === "failed" && lastUpdated < oneHourAgo; + // cell: ({ row }) => { + // // const oneHourAgo = new Date(new Date().getTime() - (60 * 60 * 1000)); + // // const lastUpdated = new Date(row.original.updated_at); + // // const canRefresh = row.original.status === "failed" && lastUpdated < oneHourAgo; + // const canReDownload = true; + // + // return ( + //
+ // + // {row.original.status} + // + // {canReDownload && ( + // { + // redownloadCheckpoint(row.original); + // }} + // className="h-4 w-4 cursor-pointer" // Adjust the size with h-x and w-x classes + // /> + // )} + //
+ // ); + // }, + }, + }, + { + accessorKey: "upload_type", + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + return {row.original.upload_type}; + }, + }, + { + accessorKey: "date", + sortingFn: "datetime", + enableSorting: true, + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => ( +
+ {getRelativeTime(row.original.updated_at)} +
+ ), + }, + // TODO: deletion and editing for future sprint + // { + // id: "actions", + // enableHiding: false, + // cell: ({ row }) => { + // const checkpoint = row.original; + // + // return ( + // + // + // + // + // + // Actions + // { + // deleteWorkflow(checkpoint.id); + // }} + // > + // Delete Workflow + // + // + // + // ); + // }, + // }, +]; + +export function CheckpointList({ data }: { data: CheckpointItemList[] }) { + const [sorting, setSorting] = React.useState([]); + const [columnFilters, setColumnFilters] = React.useState( + [] + ); + const [columnVisibility, setColumnVisibility] = + React.useState({}); + const [rowSelection, setRowSelection] = React.useState({}); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + onRowSelectionChange: setRowSelection, + state: { + sorting, + columnFilters, + columnVisibility, + rowSelection, + }, + }); + + return ( +
+
+ + table.getColumn("name")?.setFilterValue(event.target.value) + } + className="max-w-sm" + /> +
+ + Pick a checkpoint from{" "} + + civitai.com + {" "} + and place it's url here + + ), + }, + }} + /> +
+
+ + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext() + )} + + ); + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext() + )} + + ))} + + )) + ) : ( + + + No results. + + + )} + +
+ +
+
+ {table.getFilteredSelectedRowModel().rows.length} of{" "} + {table.getFilteredRowModel().rows.length} row(s) selected. +
+
+ + +
+
+
+ ); +} diff --git a/web/src/components/NavbarMenu.tsx b/web/src/components/NavbarMenu.tsx index 74cc942..81345c6 100644 --- a/web/src/components/NavbarMenu.tsx +++ b/web/src/components/NavbarMenu.tsx @@ -34,6 +34,10 @@ export function NavbarMenu({ className }: { className?: string }) { name: "API Keys", path: "/api-keys", }, + { + name: "Storage", + path: "/storage", + }, ]; return ( @@ -42,9 +46,9 @@ export function NavbarMenu({ className }: { className?: string }) { {isDesktop && ( - + {pages.map((page) => ( >(); + const [modelVersionid, setModelVersionId] = React.useState(); + const { label, isRequired, fieldProps, zodItem, fieldConfigItem } = props; + + const handleSearch = useDebouncedCallback((search) => { + const validationResult = + insertCivitaiCheckpointSchema.shape.civitai_url.safeParse(search); + if (!validationResult.success) { + console.error(validationResult.error); + // Optionally set an error state here + return; + } + + setLoading(true); + + const controller = new AbortController(); + const { url, modelVersionId: versionId } = getUrl(search); + setModelVersionId(versionId); + fetch(url, { + signal: controller.signal, + }) + .then((x) => x.json()) + .then((a) => { + const res = CivitaiModelResponse.parse(a); + console.log(a); + console.log(res); + setModelRes(res); + setLoading(false); + }); + + return () => { + controller.abort(); + setLoading(false); + }; + }, 300); + + const modifiedField = { + ...fieldProps, + // onChange: (event: React.ChangeEvent) => { + // handleSearch(event.target.value); + // }, + }; + + return ( + + {fieldConfigItem.inputProps?.showLabel && ( + + {label} + {isRequired && *} + + )} + + + + + ); +} diff --git a/web/src/db/schema.ts b/web/src/db/schema.ts index a8700fd..da3433c 100644 --- a/web/src/db/schema.ts +++ b/web/src/db/schema.ts @@ -1,3 +1,4 @@ +import { CivitaiModelResponse } from "@/types/civitai"; import { type InferSelectModel, relations } from "drizzle-orm"; import { boolean, @@ -92,7 +93,7 @@ export const workflowVersionRelations = relations( fields: [workflowVersionTable.workflow_id], references: [workflowTable.id], }), - }), + }) ); export const workflowRunStatus = pgEnum("workflow_run_status", [ @@ -141,7 +142,7 @@ export const workflowRunsTable = dbSchema.table("workflow_runs", { () => workflowVersionTable.id, { onDelete: "set null", - }, + } ), workflow_inputs: jsonb("workflow_inputs").$type>(), @@ -181,7 +182,7 @@ export const workflowRunRelations = relations( fields: [workflowRunsTable.workflow_id], references: [workflowTable.id], }), - }), + }) ); // We still want to keep the workflow run record. @@ -205,7 +206,7 @@ export const workflowOutputRelations = relations( fields: [workflowRunOutputs.run_id], references: [workflowRunsTable.id], }), - }), + }) ); // when user delete, also delete all the workflow versions @@ -238,7 +239,7 @@ export const snapshotType = z.object({ z.object({ hash: z.string(), disabled: z.boolean(), - }), + }) ), file_custom_nodes: z.array(z.any()), }); @@ -253,7 +254,7 @@ export const showcaseMedia = z.array( z.object({ url: z.string(), isCover: z.boolean().default(false), - }), + }) ); export const showcaseMediaNullable = z @@ -261,7 +262,7 @@ export const showcaseMediaNullable = z z.object({ url: z.string(), isCover: z.boolean().default(false), - }), + }) ) .nullable(); @@ -363,6 +364,89 @@ export const authRequestsTable = dbSchema.table("auth_requests", { updated_at: timestamp("updated_at").defaultNow().notNull(), }); +export const resourceUpload = pgEnum("resource_upload", [ + "started", + "success", + "failed", +]); + +export const modelUploadType = pgEnum("model_upload_type", [ + "civitai", + "huggingface", + "other", +]); + +export const checkpointTable = dbSchema.table("checkpoints", { + id: uuid("id").primaryKey().defaultRandom().notNull(), + user_id: text("user_id").references(() => usersTable.id, {}), // perhaps a "special" user_id for global checkpoints + org_id: text("org_id"), + description: text("description"), + + checkpoint_volume_id: uuid("checkpoint_volume_id") + .notNull() + .references(() => checkpointVolumeTable.id, { + onDelete: "cascade", + }) + .notNull(), + + model_name: text("model_name"), + folder_path: text("folder_path"), // in volume + + civitai_id: text("civitai_id"), + civitai_version_id: text("civitai_version_id"), + civitai_url: text("civitai_url"), + civitai_download_url: text("civitai_download_url"), + civitai_model_response: jsonb("civitai_model_response").$type< + z.infer + >(), + + hf_url: text("hf_url"), + s3_url: text("s3_url"), + user_url: text("client_url"), + + is_public: boolean("is_public").notNull().default(false), + status: resourceUpload("status").notNull().default("started"), + upload_machine_id: text("upload_machine_id"), // TODO: review if actually needed + upload_type: modelUploadType("upload_type").notNull(), + error_log: text("error_log"), + created_at: timestamp("created_at").defaultNow().notNull(), + updated_at: timestamp("updated_at").defaultNow().notNull(), +}); + +export const checkpointVolumeTable = dbSchema.table("checkpoint_volume", { + id: uuid("id").primaryKey().defaultRandom().notNull(), + user_id: text("user_id").references(() => usersTable.id, { + // onDelete: "cascade", + }), + org_id: text("org_id"), + volume_name: text("volume_name").notNull(), + created_at: timestamp("created_at").defaultNow().notNull(), + updated_at: timestamp("updated_at").defaultNow().notNull(), + disabled: boolean("disabled").default(false).notNull(), +}); + +export const checkpointRelations = relations(checkpointTable, ({ one }) => ({ + user: one(usersTable, { + fields: [checkpointTable.user_id], + references: [usersTable.id], + }), + volume: one(checkpointVolumeTable, { + fields: [checkpointTable.checkpoint_volume_id], + references: [checkpointVolumeTable.id], + }), +})); + +export const checkpointVolumeRelations = relations( + checkpointVolumeTable, + ({ many, one }) => ({ + checkpoint: many(checkpointTable), + user: one(usersTable, { + fields: [checkpointVolumeTable.user_id], + references: [usersTable.id], + }), + }) +); + export const subscriptionPlan = pgEnum("subscription_plan", [ "basic", "pro", @@ -389,9 +473,26 @@ export const subscriptionStatusTable = dbSchema.table("subscription_status", { updated_at: timestamp("updated_at").defaultNow().notNull(), }); +export const insertCivitaiCheckpointSchema = createInsertSchema( + checkpointTable, + { + civitai_url: (schema) => + schema.civitai_url + .trim() + .url({ message: "URL required" }) + .includes("civitai.com/models", { + message: "civitai.com/models link required", + }), + } +); + export type UserType = InferSelectModel; export type WorkflowType = InferSelectModel; export type MachineType = InferSelectModel; export type WorkflowVersionType = InferSelectModel; export type DeploymentType = InferSelectModel; +export type CheckpointType = InferSelectModel; +export type CheckpointVolumeType = InferSelectModel< + typeof checkpointVolumeTable +>; export type UserUsageType = InferSelectModel; diff --git a/web/src/server/addCheckpointSchema.tsx b/web/src/server/addCheckpointSchema.tsx new file mode 100644 index 0000000..b83f5a3 --- /dev/null +++ b/web/src/server/addCheckpointSchema.tsx @@ -0,0 +1,5 @@ +import { insertCivitaiCheckpointSchema } from "@/db/schema"; + +export const addCivitaiCheckpointSchema = insertCivitaiCheckpointSchema.pick({ + civitai_url: true, +}); diff --git a/web/src/server/curdCheckpoint.ts b/web/src/server/curdCheckpoint.ts new file mode 100644 index 0000000..3c01532 --- /dev/null +++ b/web/src/server/curdCheckpoint.ts @@ -0,0 +1,271 @@ +"use server"; + +import { auth } from "@clerk/nextjs"; +import { + checkpointTable, + CheckpointType, + checkpointVolumeTable, + CheckpointVolumeType, +} from "@/db/schema"; +import { withServerPromise } from "./withServerPromise"; +import { db } from "@/db/db"; +import type { z } from "zod"; +import { headers } from "next/headers"; +import { addCivitaiCheckpointSchema } from "./addCheckpointSchema"; +import { and, eq, isNull } from "drizzle-orm"; +import { CivitaiModelResponse } from "@/types/civitai"; + +export async function getCheckpoints() { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + const checkpoints = await db + .select() + .from(checkpointTable) + .where( + orgId + ? eq(checkpointTable.org_id, orgId) + // make sure org_id is null + : and( + eq(checkpointTable.user_id, userId), + isNull(checkpointTable.org_id), + ), + ); + return checkpoints; +} + +export async function getCheckpointById(id: string) { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + const checkpoint = await db + .select() + .from(checkpointTable) + .where( + and( + orgId ? eq(checkpointTable.org_id, orgId) : and( + eq(checkpointTable.user_id, userId), + isNull(checkpointTable.org_id), + ), + eq(checkpointTable.id, id), + ), + ); + return checkpoint[0]; +} + +export async function getCheckpointVolumes() { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + const volume = await db + .select() + .from(checkpointVolumeTable) + .where( + and( + orgId + ? eq(checkpointVolumeTable.org_id, orgId) + // make sure org_id is null + : and( + eq(checkpointVolumeTable.user_id, userId), + isNull(checkpointVolumeTable.org_id), + ), + eq(checkpointVolumeTable.disabled, false), + ), + ); + return volume; +} + +export async function retrieveCheckpointVolumes() { + let volumes = await getCheckpointVolumes(); + if (volumes.length === 0) { + // create volume if not already created + volumes = await addCheckpointVolume(); + } + return volumes; +} + +export async function addCheckpointVolume() { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + + // Insert the new checkpointVolume into the checkpointVolumeTable + const insertedVolume = await db + .insert(checkpointVolumeTable) + .values({ + user_id: userId, + org_id: orgId, + volume_name: `checkpoints_${userId}`, + // created_at and updated_at will be set to current timestamp by default + disabled: false, // Default value + }) + .returning(); // Returns the inserted row + return insertedVolume; +} + +function getUrl(civitai_url: string) { + // expect to be a URL to be https://civitai.com/models/36520 + // possiblity with slugged name and query-param modelVersionId + const baseUrl = "https://civitai.com/api/v1/models/"; + const url = new URL(civitai_url); + const pathSegments = url.pathname.split("/"); + const modelId = pathSegments[pathSegments.indexOf("models") + 1]; + const modelVersionId = url.searchParams.get("modelVersionId"); + + return { url: baseUrl + modelId, modelVersionId }; +} + +export const addCivitaiCheckpoint = withServerPromise( + async (data: z.infer) => { + const { userId, orgId } = auth(); + + if (!data.civitai_url) return { error: "no civitai_url" }; + if (!userId) return { error: "No user id" }; + + const { url, modelVersionId } = getUrl(data?.civitai_url); + const civitaiModelRes = await fetch(url) + .then((x) => x.json()) + .then((a) => { + console.log(a); + return CivitaiModelResponse.parse(a); + }); + + if (civitaiModelRes?.modelVersions?.length === 0) { + return; // no versions to download + } + + let selectedModelVersion; + let selectedModelVersionId: string | null = modelVersionId; + if (!selectedModelVersionId) { + selectedModelVersion = civitaiModelRes.modelVersions[0]; + selectedModelVersionId = civitaiModelRes.modelVersions[0].id.toString(); + } else { + selectedModelVersion = civitaiModelRes.modelVersions.find((version) => + version.id.toString() === selectedModelVersionId + ); + if (!selectedModelVersion) { + return; // version id is wrong + } + selectedModelVersionId = selectedModelVersion?.id.toString(); + } + + const checkpointVolumes = await getCheckpointVolumes(); + let cVolume; + if (checkpointVolumes.length === 0) { + const volume = await addCheckpointVolume(); + cVolume = volume[0]; + } else { + cVolume = checkpointVolumes[0]; + } + + const a = await db + .insert(checkpointTable) + .values({ + user_id: userId, + org_id: orgId, + upload_type: "civitai", + model_name: selectedModelVersion.files[0].name, + civitai_id: civitaiModelRes.id.toString(), + civitai_version_id: selectedModelVersionId, + civitai_url: data.civitai_url, + civitai_download_url: selectedModelVersion.files[0].downloadUrl, + civitai_model_response: civitaiModelRes, + checkpoint_volume_id: cVolume.id, + updated_at: new Date(), + }) + .returning(); + + const b = a[0]; + + await uploadCheckpoint(data, b, cVolume); + // redirect(`/checkpoints/${b.id}`); + }, +); + +// export const redownloadCheckpoint = withServerPromise( +// async (data: CheckpointItemList) => { +// const { userId } = auth(); +// if (!userId) return { error: "No user id" }; +// +// const checkpointVolumes = await getCheckpointVolumes(); +// let cVolume; +// if (checkpointVolumes.length === 0) { +// const volume = await addCheckpointVolume(); +// cVolume = volume[0]; +// } else { +// cVolume = checkpointVolumes[0]; +// } +// +// console.log("data"); +// console.log(data); +// +// const a = await db +// .update(checkpointTable) +// .set({ +// // status: "started", +// // updated_at: new Date(), +// }) +// .returning(); +// +// const b = a[0]; +// +// console.log("b"); +// console.log(b); +// +// await uploadCheckpoint(data, b, cVolume); +// // redirect(`/checkpoints/${b.id}`); +// }, +// ); + +async function uploadCheckpoint( + data: z.infer, + c: CheckpointType, + v: CheckpointVolumeType, +) { + const headersList = headers(); + + const domain = headersList.get("x-forwarded-host") || ""; + const protocol = headersList.get("x-forwarded-proto") || ""; + + if (domain === "") { + throw new Error("No domain"); + } + + // Call remote builder + const result = await fetch( + `${process.env.MODAL_BUILDER_URL!}/upload-volume`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + download_url: c.civitai_download_url, + volume_name: v.volume_name, + volume_id: v.id, + checkpoint_id: c.id, + callback_url: `${protocol}://${domain}/api/volume-upload`, + upload_type: "checkpoint" + }), + }, + ); + + if (!result.ok) { + const error_log = await result.text(); + await db + .update(checkpointTable) + .set({ + ...data, + status: "failed", + error_log: error_log, + }) + .where(eq(checkpointTable.id, c.id)); + throw new Error(`Error: ${result.statusText} ${error_log}`); + } else { + // setting the build machine id + const json = await result.json(); + await db + .update(checkpointTable) + .set({ + ...data, + upload_machine_id: json.build_machine_instance_id, + }) + .where(eq(checkpointTable.id, c.id)); + } +} diff --git a/web/src/server/curdMachine.ts b/web/src/server/curdMachine.ts index b59415f..77fea3b 100644 --- a/web/src/server/curdMachine.ts +++ b/web/src/server/curdMachine.ts @@ -15,6 +15,7 @@ import { headers } from "next/headers"; import { redirect } from "next/navigation"; import "server-only"; import type { z } from "zod"; +import { retrieveCheckpointVolumes } from "./curdCheckpoint"; export async function getMachines() { const { userId, orgId } = auth(); @@ -189,6 +190,7 @@ async function _buildMachine( throw new Error("No domain"); } + const volumes = await retrieveCheckpointVolumes(); // Call remote builder const result = await fetch(`${process.env.MODAL_BUILDER_URL!}/create`, { method: "POST", @@ -202,6 +204,7 @@ async function _buildMachine( callback_url: `${protocol}://${domain}/api/machine-built`, models: data.models, //JSON.parse(data.models as string), gpu: data.gpu && data.gpu.length > 0 ? data.gpu : "T4", + checkpoint_volume_name: volumes[0].volume_name, }), }); diff --git a/web/src/server/getAllUserCheckpoints.tsx b/web/src/server/getAllUserCheckpoints.tsx new file mode 100644 index 0000000..a12a668 --- /dev/null +++ b/web/src/server/getAllUserCheckpoints.tsx @@ -0,0 +1,41 @@ +import { db } from "@/db/db"; +import { + checkpointTable, +} from "@/db/schema"; +import { auth } from "@clerk/nextjs"; +import { and, desc, eq, isNull } from "drizzle-orm"; + +export async function getAllUserCheckpoints() { + const { userId, orgId } = await auth(); + + if (!userId) { + return null; + } + + const checkpoints = await db.query.checkpointTable.findMany({ + with: { + user: { + columns: { + name: true, + }, + }, + }, + columns: { + id: true, + updated_at: true, + model_name: true, + civitai_url: true, + civitai_model_response: true, + is_public: true, + upload_type: true, + status: true, + }, + orderBy: desc(checkpointTable.updated_at), + where: + orgId != undefined + ? eq(checkpointTable.org_id, orgId) + : and(eq(checkpointTable.user_id, userId), isNull(checkpointTable.org_id)), + }); + + return checkpoints; +} diff --git a/web/src/types/civitai.ts b/web/src/types/civitai.ts new file mode 100644 index 0000000..5e57ed0 --- /dev/null +++ b/web/src/types/civitai.ts @@ -0,0 +1,129 @@ +import { z } from "zod"; + +// from chatgpt https://chat.openai.com/share/4985d20b-30b1-4a28-87f6-6ebf84a1040e + +export const creatorSchema = z.object({ + username: z.string().nullish(), + image: z.string().url().nullish(), +}); + +export const fileMetadataSchema = z.object({ + fp: z.string().nullish(), + size: z.string().nullish(), + format: z.string().nullish(), +}); + +export const fileSchema = z.object({ + id: z.number(), + sizeKB: z.number().nullish(), + name: z.string(), + type: z.string().nullish(), + metadata: fileMetadataSchema.nullish(), + pickleScanResult: z.string().nullish(), + pickleScanMessage: z.string().nullable(), + virusScanResult: z.string().nullish(), + virusScanMessage: z.string().nullable(), + scannedAt: z.string().nullish(), + hashes: z.record(z.string()).nullish(), + downloadUrl: z.string().url(), + primary: z.boolean().nullish(), +}); + +export const imageMetadataSchema = z.object({ + hash: z.string(), + width: z.number(), + height: z.number(), +}); + +export const imageMetaSchema = z.object({ + ENSD: z.string().nullish(), + Size: z.string().nullish(), + seed: z.number().nullish(), + Model: z.string().nullish(), + steps: z.number().nullish(), + hashes: z.record(z.string()).nullish(), + prompt: z.string().nullish(), + sampler: z.string().nullish(), + cfgScale: z.number().nullish(), + ClipSkip: z.number().nullish(), + resources: z.array( + z.object({ + hash: z.string().nullish(), + name: z.string(), + type: z.string(), + weight: z.number().nullish(), + }), + ).nullish(), + ModelHash: z.string().nullish(), + HiresSteps: z.string().nullish(), + HiresUpscale: z.string().nullish(), + HiresUpscaler: z.string().nullish(), + negativePrompt: z.string(), + DenoisingStrength: z.number().nullish(), +}); + +// NOTE: this definition is all over the place +// export const imageSchema = z.object({ +// url: z.string().url().nullish(), +// nsfw: z.enum(["None", "Soft", "Mature"]).nullish(), +// width: z.number().nullish(), +// height: z.number().nullish(), +// hash: z.string().nullish(), +// type: z.string().nullish(), +// metadata: imageMetadataSchema.nullish(), +// meta: imageMetaSchema.nullish(), +// }); + +export const modelVersionSchema = z.object({ + id: z.number(), + modelId: z.number(), + name: z.string(), + createdAt: z.string().nullish(), + updatedAt: z.string().nullish(), + // status: z.enum(["Published", "Unpublished"]).nullish(), + status: z.string().nullish(), + publishedAt: z.string().nullish(), + trainedWords: z.array(z.string()).nullable(), + trainingStatus: z.string().nullable(), + trainingDetails: z.string().nullable(), + baseModel: z.string().nullish(), + baseModelType: z.string().nullish(), + earlyAccessTimeFrame: z.number().nullish(), + description: z.string().nullable(), + vaeId: z.number().nullable(), + stats: z.object({ + downloadCount: z.number(), + ratingCount: z.number(), + rating: z.number(), + }).nullish(), + files: z.array(fileSchema), + images: z.array(z.any()).nullish(), + downloadUrl: z.string().url(), +}); + +export const statsSchema = z.object({ + downloadCount: z.number(), + favoriteCount: z.number(), + commentCount: z.number(), + ratingCount: z.number(), + rating: z.number(), + tippedAmountCount: z.number(), +}); + +export const CivitaiModelResponse = z.object({ + id: z.number(), + name: z.string().nullish(), + description: z.string().nullish(), + // type: z.enum(["Checkpoint", "Lora"]), // TODO: this will be important to know + type: z.string(), + poi: z.boolean().nullish(), + nsfw: z.boolean().nullish(), + allowNoCredit: z.boolean().nullish(), + allowCommercialUse: z.string().nullish(), + allowDerivatives: z.boolean().nullish(), + allowDifferentLicense: z.boolean().nullish(), + stats: statsSchema.nullish(), + creator: creatorSchema.nullish(), + tags: z.array(z.string()).nullish(), + modelVersions: z.array(modelVersionSchema), +});