diff --git a/builder/modal-builder/src/main.py b/builder/modal-builder/src/main.py index c0e0170..dfe5c0c 100644 --- a/builder/modal-builder/src/main.py +++ b/builder/modal-builder/src/main.py @@ -224,6 +224,20 @@ async def websocket_endpoint(websocket: WebSocket, machine_id: str): # return {"Hello": "World"} +class UploadBody(BaseModel): + download_url: str + volume_name: str + # callback_url: str + +@app.post("/upload_volume") +async def upload_checkpoint(body: UploadBody): + download_url = body.download_url + volume_name = body.download_url + # callback_url = body.callback_url + # check that thi + return + + @app.post("/create") async def create_machine(item: Item): global last_activity_time diff --git a/builder/modal-builder/src/template/data/insert_models.py b/builder/modal-builder/src/template/data/insert_models.py index c1792b3..a189bdd 100644 --- a/builder/modal-builder/src/template/data/insert_models.py +++ b/builder/modal-builder/src/template/data/insert_models.py @@ -3,9 +3,9 @@ This is a standalone script to download models into a modal Volume using civitai Example Usage `modal run insert_models::insert_model --civitai-url https://civitai.com/models/36520/ghostmix` -This inserts an individual model from a civitai url (public not API url) +This inserts an individual model from a civitai url -`modal run insert_models::insert_models` +`modal run insert_models::insert_models_civitai_api` This inserts a bunch of models based on the models retrieved by civitai civitai's API reference https://github.com/civitai/civitai/wiki/REST-API-Reference @@ -13,27 +13,24 @@ civitai's API reference https://github.com/civitai/civitai/wiki/REST-API-Referen import modal import subprocess import requests +import json stub = modal.Stub() # NOTE: volume name can be variable -volume = modal.Volume.persisted("private-model-store") +volume = modal.Volume.persisted("rah") model_store_path = "/vol/models" MODEL_ROUTE = "models" image = ( modal.Image.debian_slim().apt_install("wget").pip_install("requests") ) -@stub.function(volumes={model_store_path: volume}, gpu="any", image=image, timeout=600) -def download_model(model): - # wget https://civitai.com/api/download/models/{modelVersionId} --content-disposition - # model_id = model['modelVersions'][0]['id'] - # download_url = f"https://civitai.com/api/download/models/{model_id}" - - download_url = model['modelVersions'][0]['downloadUrl'] +@stub.function(volumes={model_store_path: volume}, image=image, timeout=50000, gpu=None) +def download_model(download_url): + print(download_url) subprocess.run(["wget", download_url, "--content-disposition", "-P", model_store_path]) subprocess.run(["ls", "-la", model_store_path]) - volume.commit() + volume.commit() # file is raw output from Civitai API https://github.com/civitai/civitai/wiki/REST-API-Reference @@ -52,40 +49,53 @@ def get_civitai_models(model_type: str, sort: str = "Highest Rated", page: int = @stub.function() def get_civitai_model_url(civitai_url: str): # Validate the URL - if not civitai_url.startswith("https://civitai.com/models/"): + + if civitai_url.startswith("https://civitai.com/api/"): + api_url = civitai_url + elif civitai_url.startswith("https://civitai.com/models/"): + try: + model_id = civitai_url.split("/")[4] + int(model_id) + except (IndexError, ValueError): + return None + api_url = f"https://civitai.com/api/v1/models/{model_id}" + else: return "Error: URL must be from civitai.com and contain /models/" - # Extract the model ID - try: - model_id = civitai_url.split("/")[4] - int(model_id) # Check if the ID is an integer - except (IndexError, ValueError): - return None #Error: Invalid model ID in URL - - # Make the API request - api_url = f"https://civitai.com/api/v1/models/{model_id}" response = requests.get(api_url) - # Check for successful response if response.status_code != 200: return f"Error: Unable to fetch data from {api_url}" - # Return the response data return response.json() @stub.local_entrypoint() -def insert_models(type: str = "Checkpoint", sort = "Highest Rated", page: int = 1): +def insert_models_civitai_api(type: str = "Checkpoint", sort = "Highest Rated", page: int = 1): civitai_models = get_civitai_models.local(type, sort, page) if civitai_models: - for _ in download_model.map(civitai_models['items'][1:]): + for _ in download_model.map(map(lambda model: model['modelVersions'][0]['downloadUrl'], civitai_models['items'])): pass else: print("Failed to retrieve models.") @stub.local_entrypoint() def insert_model(civitai_url: str): - civitai_model = get_civitai_model_url.local(civitai_url) - if civitai_model: - download_model.remote(civitai_model) + if civitai_url.startswith("'https://civitai.com/api/download/models/"): + download_url = civitai_url + else: + civitai_model = get_civitai_model_url.local(civitai_url) + if civitai_model: + download_url = civitai_model['modelVersions'][0]['downloadUrl'] + else: + return "invalid URL" + + download_model.remote(download_url) + +@stub.local_entrypoint() +def simple_download(): + download_urls = ['https://civitai.com/api/download/models/119057', 'https://civitai.com/api/download/models/130090', 'https://civitai.com/api/download/models/31859', 'https://civitai.com/api/download/models/128713', 'https://civitai.com/api/download/models/179657', 'https://civitai.com/api/download/models/143906', 'https://civitai.com/api/download/models/9208', 'https://civitai.com/api/download/models/136078', 'https://civitai.com/api/download/models/134065', 'https://civitai.com/api/download/models/288775', 'https://civitai.com/api/download/models/95263', 'https://civitai.com/api/download/models/288982', 'https://civitai.com/api/download/models/87153', 'https://civitai.com/api/download/models/10638', 'https://civitai.com/api/download/models/263809', 'https://civitai.com/api/download/models/130072', 'https://civitai.com/api/download/models/117019', 'https://civitai.com/api/download/models/95256', 'https://civitai.com/api/download/models/197181', 'https://civitai.com/api/download/models/256915', 'https://civitai.com/api/download/models/118945', 'https://civitai.com/api/download/models/125843', 'https://civitai.com/api/download/models/179015', 'https://civitai.com/api/download/models/245598', 'https://civitai.com/api/download/models/223670', 'https://civitai.com/api/download/models/90072', 'https://civitai.com/api/download/models/290817', 'https://civitai.com/api/download/models/154097', 'https://civitai.com/api/download/models/143497', 'https://civitai.com/api/download/models/5637'] + + for _ in download_model.map(download_urls): + pass diff --git a/builder/modal-builder/src/volume-builder/app.py b/builder/modal-builder/src/volume-builder/app.py new file mode 100644 index 0000000..186b051 --- /dev/null +++ b/builder/modal-builder/src/volume-builder/app.py @@ -0,0 +1,54 @@ +import modal +from config import config +import os +import uuid +import subprocess + +stub = modal.Stub() + +base_path = "/volumes" + +# Volume names may only contain alphanumeric characters, dashes, periods, and underscores, and must be less than 64 characters in length. +def is_valid_name(name: str) -> bool: + allowed_characters = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._") + return 0 < len(name) <= 64 and all(char in allowed_characters for char in name) + +def create_volumes(volume_names): + path_to_vol = {} + vol_to_path = {} + for volume_name in volume_names.keys(): + if not is_valid_name(volume_name): + pass + modal_volume = modal.Volume.persisted(volume_name) + volume_path = create_volume_path(base_path) + path_to_vol[volume_path] = modal_volume + vol_to_path[volume_name] = volume_path + + return (path_to_vol, vol_to_path) + +def create_volume_path(base_path: str): + random_path = str(uuid.uuid4()) + return os.path.join(base_path, random_path) + +vol_name_to_links = config["volume_names"] +(path_to_vol, vol_name_to_path) = create_volumes(vol_name_to_links) +image = ( + modal.Image.debian_slim().apt_install("wget").pip_install("requests") +) + +print(vol_name_to_links) +print(path_to_vol) +print(vol_name_to_path) + +@stub.function(volumes=path_to_vol, image=image, timeout=5000, gpu=None) +def download_model(volume_name, download_url): + model_store_path = vol_name_to_path[volume_name] + subprocess.run(["wget", download_url, "--content-disposition", "-P", model_store_path]) + subprocess.run(["ls", "-la", model_store_path]) + path_to_vol[model_store_path].commit() + +@stub.local_entrypoint() +def simple_download(): + print(vol_name_to_links) + print([(vol_name, link) for vol_name,link in vol_name_to_links.items()]) + list(download_model.starmap([(vol_name, link) for vol_name,link in vol_name_to_links.items()])) diff --git a/builder/modal-builder/src/volume-builder/config.py b/builder/modal-builder/src/volume-builder/config.py new file mode 100644 index 0000000..6f4b373 --- /dev/null +++ b/builder/modal-builder/src/volume-builder/config.py @@ -0,0 +1,5 @@ +config = { + "volume_names": { + "eg1": "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose (1).png" + }, +} diff --git a/web/drizzle/0031_common_deathbird.sql b/web/drizzle/0031_common_deathbird.sql new file mode 100644 index 0000000..26fc4f3 --- /dev/null +++ b/web/drizzle/0031_common_deathbird.sql @@ -0,0 +1,62 @@ +DO $$ BEGIN + CREATE TYPE "model_upload_type" AS ENUM('civitai', 'huggingface', 'other'); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + CREATE TYPE "resource_upload" AS ENUM('started', 'failed', 'succeded'); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "comfyui_deploy"."checkpoints" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "user_id" text, + "org_id" text, + "description" text, + "checkpoint_volume_id" uuid NOT NULL, + "model_name" text, + "civitai_id" text, + "civitai_version_id" text, + "civitai_url" text, + "civitai_download_url" text, + "civitai_model_response" jsonb, + "hf_url" text, + "s3_url" text, + "client_url" text, + "is_public" boolean DEFAULT false NOT NULL, + "status" "resource_upload" DEFAULT 'started' NOT NULL, + "upload_machine_id" text, + "upload_type" "model_upload_type" NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "comfyui_deploy"."checkpointVolumeTable" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "user_id" text, + "org_id" text, + "volume_name" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL, + "disabled" boolean DEFAULT false NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "comfyui_deploy"."checkpoints" ADD CONSTRAINT "checkpoints_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "comfyui_deploy"."users"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "comfyui_deploy"."checkpoints" ADD CONSTRAINT "checkpoints_checkpoint_volume_id_workflow_runs_id_fk" FOREIGN KEY ("checkpoint_volume_id") REFERENCES "comfyui_deploy"."workflow_runs"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "comfyui_deploy"."checkpointVolumeTable" ADD CONSTRAINT "checkpointVolumeTable_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "comfyui_deploy"."users"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; diff --git a/web/drizzle/meta/0031_snapshot.json b/web/drizzle/meta/0031_snapshot.json new file mode 100644 index 0000000..de04e7a --- /dev/null +++ b/web/drizzle/meta/0031_snapshot.json @@ -0,0 +1,1004 @@ +{ + "id": "66dbc84a-6cd8-4692-9d24-fdcac227b23c", + "prevId": "db06ea66-92c2-4ebe-93c1-6cb8a90ccd8b", + "version": "5", + "dialect": "pg", + "tables": { + "api_keys": { + "name": "api_keys", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_users_id_fk": { + "name": "api_keys_user_id_users_id_fk", + "tableFrom": "api_keys", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_key_unique": { + "name": "api_keys_key_unique", + "nullsNotDistinct": false, + "columns": [ + "key" + ] + } + } + }, + "checkpoints": { + "name": "checkpoints", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checkpoint_volume_id": { + "name": "checkpoint_volume_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_id": { + "name": "civitai_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_version_id": { + "name": "civitai_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_url": { + "name": "civitai_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_download_url": { + "name": "civitai_download_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "civitai_model_response": { + "name": "civitai_model_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hf_url": { + "name": "hf_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_url": { + "name": "s3_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_url": { + "name": "client_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_public": { + "name": "is_public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "resource_upload", + "primaryKey": false, + "notNull": true, + "default": "'started'" + }, + "upload_machine_id": { + "name": "upload_machine_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upload_type": { + "name": "upload_type", + "type": "model_upload_type", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "checkpoints_user_id_users_id_fk": { + "name": "checkpoints_user_id_users_id_fk", + "tableFrom": "checkpoints", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "checkpoints_checkpoint_volume_id_workflow_runs_id_fk": { + "name": "checkpoints_checkpoint_volume_id_workflow_runs_id_fk", + "tableFrom": "checkpoints", + "tableTo": "workflow_runs", + "columnsFrom": [ + "checkpoint_volume_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "checkpointVolumeTable": { + "name": "checkpointVolumeTable", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "volume_name": { + "name": "volume_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": { + "checkpointVolumeTable_user_id_users_id_fk": { + "name": "checkpointVolumeTable_user_id_users_id_fk", + "tableFrom": "checkpointVolumeTable", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "deployments": { + "name": "deployments", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow_version_id": { + "name": "workflow_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "machine_id": { + "name": "machine_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "showcase_media": { + "name": "showcase_media", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "environment": { + "name": "environment", + "type": "deployment_environment", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "deployments_user_id_users_id_fk": { + "name": "deployments_user_id_users_id_fk", + "tableFrom": "deployments", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deployments_workflow_version_id_workflow_versions_id_fk": { + "name": "deployments_workflow_version_id_workflow_versions_id_fk", + "tableFrom": "deployments", + "tableTo": "workflow_versions", + "columnsFrom": [ + "workflow_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployments_workflow_id_workflows_id_fk": { + "name": "deployments_workflow_id_workflows_id_fk", + "tableFrom": "deployments", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "deployments_machine_id_machines_id_fk": { + "name": "deployments_machine_id_machines_id_fk", + "tableFrom": "deployments", + "tableTo": "machines", + "columnsFrom": [ + "machine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "machines": { + "name": "machines", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "auth_token": { + "name": "auth_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "machine_type", + "primaryKey": false, + "notNull": true, + "default": "'classic'" + }, + "status": { + "name": "status", + "type": "machine_status", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "models": { + "name": "models", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "gpu": { + "name": "gpu", + "type": "machine_gpu", + "primaryKey": false, + "notNull": false + }, + "build_machine_instance_id": { + "name": "build_machine_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "build_log": { + "name": "build_log", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "machines_user_id_users_id_fk": { + "name": "machines_user_id_users_id_fk", + "tableFrom": "machines", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "users": { + "name": "users", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_run_outputs": { + "name": "workflow_run_outputs", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_run_outputs_run_id_workflow_runs_id_fk": { + "name": "workflow_run_outputs_run_id_workflow_runs_id_fk", + "tableFrom": "workflow_run_outputs", + "tableTo": "workflow_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_runs": { + "name": "workflow_runs", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workflow_version_id": { + "name": "workflow_version_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "workflow_inputs": { + "name": "workflow_inputs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "machine_id": { + "name": "machine_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "workflow_run_origin", + "primaryKey": false, + "notNull": true, + "default": "'api'" + }, + "status": { + "name": "status", + "type": "workflow_run_status", + "primaryKey": false, + "notNull": true, + "default": "'not-started'" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_runs_workflow_version_id_workflow_versions_id_fk": { + "name": "workflow_runs_workflow_version_id_workflow_versions_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "workflow_versions", + "columnsFrom": [ + "workflow_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_runs_workflow_id_workflows_id_fk": { + "name": "workflow_runs_workflow_id_workflows_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_runs_machine_id_machines_id_fk": { + "name": "workflow_runs_machine_id_machines_id_fk", + "tableFrom": "workflow_runs", + "tableTo": "machines", + "columnsFrom": [ + "machine_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflows": { + "name": "workflows", + "schema": "comfyui_deploy", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflows_user_id_users_id_fk": { + "name": "workflows_user_id_users_id_fk", + "tableFrom": "workflows", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "workflow_versions": { + "name": "workflow_versions", + "schema": "comfyui_deploy", + "columns": { + "workflow_id": { + "name": "workflow_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workflow": { + "name": "workflow", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workflow_api": { + "name": "workflow_api", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_versions_workflow_id_workflows_id_fk": { + "name": "workflow_versions_workflow_id_workflows_id_fk", + "tableFrom": "workflow_versions", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "deployment_environment": { + "name": "deployment_environment", + "values": { + "staging": "staging", + "production": "production", + "public-share": "public-share" + } + }, + "machine_gpu": { + "name": "machine_gpu", + "values": { + "T4": "T4", + "A10G": "A10G", + "A100": "A100" + } + }, + "machine_status": { + "name": "machine_status", + "values": { + "ready": "ready", + "building": "building", + "error": "error" + } + }, + "machine_type": { + "name": "machine_type", + "values": { + "classic": "classic", + "runpod-serverless": "runpod-serverless", + "modal-serverless": "modal-serverless", + "comfy-deploy-serverless": "comfy-deploy-serverless" + } + }, + "model_upload_type": { + "name": "model_upload_type", + "values": { + "civitai": "civitai", + "huggingface": "huggingface", + "other": "other" + } + }, + "resource_upload": { + "name": "resource_upload", + "values": { + "started": "started", + "failed": "failed", + "succeded": "succeded" + } + }, + "workflow_run_origin": { + "name": "workflow_run_origin", + "values": { + "manual": "manual", + "api": "api", + "public-share": "public-share" + } + }, + "workflow_run_status": { + "name": "workflow_run_status", + "values": { + "not-started": "not-started", + "running": "running", + "uploading": "uploading", + "success": "success", + "failed": "failed" + } + } + }, + "schemas": { + "comfyui_deploy": "comfyui_deploy" + }, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/web/drizzle/meta/_journal.json b/web/drizzle/meta/_journal.json index 3bfcc76..63aa103 100644 --- a/web/drizzle/meta/_journal.json +++ b/web/drizzle/meta/_journal.json @@ -218,6 +218,13 @@ "when": 1705716303820, "tag": "0030_kind_doorman", "breakpoints": true + }, + { + "idx": 31, + "version": "5", + "when": 1705963548821, + "tag": "0031_common_deathbird", + "breakpoints": true } ] } \ No newline at end of file diff --git a/web/src/app/(app)/api/volume-updated/route.ts b/web/src/app/(app)/api/volume-updated/route.ts new file mode 100644 index 0000000..386c00b --- /dev/null +++ b/web/src/app/(app)/api/volume-updated/route.ts @@ -0,0 +1,50 @@ +import { parseDataSafe } from "../../../../lib/parseDataSafe"; +import { db } from "@/db/db"; +import { checkpointTable, machinesTable } from "@/db/schema"; +import { eq } from "drizzle-orm"; +import { NextResponse } from "next/server"; +import { z } from "zod"; + +const Request = z.object({ + machine_id: z.string(), + endpoint: z.string().optional(), + build_log: z.string().optional(), +}); + +export async function POST(request: Request) { + const [data, error] = await parseDataSafe(Request, request); + if (!data || error) return error; + + // console.log(data); + + const { machine_id, endpoint, build_log } = data; + + if (endpoint) { + await db + .update(checkpointTable) + .set({ + // status: "ready", + // endpoint: endpoint, + // build_log: build_log, + }) + .where(eq(machinesTable.id, machine_id)); + } else { + // console.log(data); + await db + .update(machinesTable) + .set({ + // status: "error", + // build_log: build_log, + }) + .where(eq(machinesTable.id, machine_id)); + } + + return NextResponse.json( + { + message: "success", + }, + { + status: 200, + } + ); +} diff --git a/web/src/app/(app)/storage/loading.tsx b/web/src/app/(app)/storage/loading.tsx new file mode 100644 index 0000000..9ff4783 --- /dev/null +++ b/web/src/app/(app)/storage/loading.tsx @@ -0,0 +1,9 @@ +"use client"; + +import { LoadingPageWrapper } from "@/components/LoadingWrapper"; +import { usePathname } from "next/navigation"; + +export default function Loading() { + const pathName = usePathname(); + return ; +} diff --git a/web/src/app/(app)/storage/page.tsx b/web/src/app/(app)/storage/page.tsx new file mode 100644 index 0000000..ce04c33 --- /dev/null +++ b/web/src/app/(app)/storage/page.tsx @@ -0,0 +1,35 @@ +import { setInitialUserData } from "../../../lib/setInitialUserData"; +import { auth } from "@clerk/nextjs"; +import { clerkClient } from "@clerk/nextjs/server"; +import { CheckpointList } from "@/components/CheckpointList" +import { getAllUserCheckpoints } from "@/server/getAllUserCheckpoints"; + +export default function Page() { + return ; +} + +async function CheckpointListServer() { + const { userId } = auth(); + + if (!userId) { + return
No auth
; + } + + const user = await clerkClient.users.getUser(userId); + + if (!user) { + await setInitialUserData(userId); + } + + const checkpoints = await getAllUserCheckpoints() + + if (!checkpoints) { + return
No checkpoints found
; + } + + return ( +
+ +
+ ); +} diff --git a/web/src/components/CheckpointList.tsx b/web/src/components/CheckpointList.tsx new file mode 100644 index 0000000..93cea9d --- /dev/null +++ b/web/src/components/CheckpointList.tsx @@ -0,0 +1,315 @@ +"use client"; + +import { getRelativeTime } from "../lib/getRelativeTime"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { InsertModal, UpdateModal } from "./InsertModal"; +import { Input } from "@/components/ui/input"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import type { getAllUserCheckpoints } from "@/server/getAllUserCheckpoints"; +import type { + ColumnDef, + ColumnFiltersState, + SortingState, + VisibilityState, +} from "@tanstack/react-table"; +import { + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { ArrowUpDown, MoreHorizontal } from "lucide-react"; +import * as React from "react"; +import { insertCivitaiCheckpointSchema } from "@/db/schema"; +import { addCivitaiCheckpoint } from "@/server/curdCheckpoint"; +import { addCivitaiCheckpointSchema } from "@/server/addCheckpointSchema"; + +export type CheckpointItemList = NonNullable< + Awaited> +>[0]; + +export const columns: ColumnDef[] = [ + { + accessorKey: "id", + id: "select", + header: ({ table }) => ( + table.toggleAllPageRowsSelected(!!value)} + aria-label="Select all" + /> + ), + cell: ({ row }) => ( + row.toggleSelected(!!value)} + aria-label="Select row" + /> + ), + enableSorting: false, + enableHiding: false, + }, + { + accessorKey: "name", + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + const checkpoint = row.original; + return ( + + {row.original.model_name} + + {} + {checkpoint.is_public + ? Public + : Private} + + ); + }, + }, + { + accessorKey: "creator", + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + // return {row?.original?.user?.name ? row.original.user.name : "Public"}; + }, + }, + { + accessorKey: "date", + sortingFn: "datetime", + enableSorting: true, + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => ( +
+ {getRelativeTime(row.original.updated_at)} +
+ ), + }, + // { + // id: "actions", + // enableHiding: false, + // cell: ({ row }) => { + // const checkpoint = row.original; + // + // return ( + // + // + // + // + // + // Actions + // { + // deleteWorkflow(checkpoint.id); + // }} + // > + // Delete Workflow + // + // + // + // ); + // }, + // }, +]; + +export function CheckpointList({ data }: { data: CheckpointItemList[] }) { + const [sorting, setSorting] = React.useState([]); + const [columnFilters, setColumnFilters] = React.useState( + [], + ); + const [columnVisibility, setColumnVisibility] = React.useState< + VisibilityState + >({}); + const [rowSelection, setRowSelection] = React.useState({}); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + onRowSelectionChange: setRowSelection, + state: { + sorting, + columnFilters, + columnVisibility, + rowSelection, + }, + }); + + return ( +
+
+ + table.getColumn("name")?.setFilterValue(event.target.value)} + className="max-w-sm" + /> +
+ + Pick a checkpoint from{" "} + + civitai.com + {" "} + and place it's url here + + ), + }, + }} + /> +
+
+ + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder ? null : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ); + })} + + ))} + + + {table.getRowModel().rows?.length + ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + + ))} + + )) + ) + : ( + + + No results. + + + )} + +
+ +
+
+ {table.getFilteredSelectedRowModel().rows.length} of{" "} + {table.getFilteredRowModel().rows.length} row(s) selected. +
+
+ + +
+
+
+ ); +} diff --git a/web/src/components/NavbarMenu.tsx b/web/src/components/NavbarMenu.tsx index 74cc942..81345c6 100644 --- a/web/src/components/NavbarMenu.tsx +++ b/web/src/components/NavbarMenu.tsx @@ -34,6 +34,10 @@ export function NavbarMenu({ className }: { className?: string }) { name: "API Keys", path: "/api-keys", }, + { + name: "Storage", + path: "/storage", + }, ]; return ( @@ -42,9 +46,9 @@ export function NavbarMenu({ className }: { className?: string }) { {isDesktop && ( - + {pages.map((page) => ( + models: z.infer, ): z.infer { return { models: models.items.flatMap((item) => { @@ -241,8 +241,9 @@ function getUrl(search?: string) { export function CivitaiModelRegistry({ field, }: Pick) { - const [modelList, setModelList] = - React.useState>(); + const [modelList, setModelList] = React.useState< + z.infer + >(); const [loading, setLoading] = React.useState(false); @@ -301,8 +302,9 @@ export function CivitaiModelRegistry({ export function ComfyUIManagerModelRegistry({ field, }: Pick) { - const [modelList, setModelList] = - React.useState>(); + const [modelList, setModelList] = React.useState< + z.infer + >(); React.useEffect(() => { const controller = new AbortController(); @@ -310,7 +312,7 @@ export function ComfyUIManagerModelRegistry({ "https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/model-list.json", { signal: controller.signal, - } + }, ) .then((x) => x.json()) .then((a) => { @@ -353,14 +355,14 @@ export function ModelSelector({ if ( prevSelectedModels.some( (selectedModel) => - selectedModel.url + selectedModel.name === model.url + model.name + selectedModel.url + selectedModel.name === model.url + model.name, ) ) { field.onChange( prevSelectedModels.filter( (selectedModel) => - selectedModel.url + selectedModel.name !== model.url + model.name - ) + selectedModel.url + selectedModel.name !== model.url + model.name, + ), ); } else { field.onChange([...prevSelectedModels, model]); @@ -408,10 +410,10 @@ export function ModelSelector({ className={cn( "ml-auto h-4 w-4", value.some( - (selectedModel) => selectedModel.url === model.url - ) + (selectedModel) => selectedModel.url === model.url, + ) ? "opacity-100" - : "opacity-0" + : "opacity-0", )} /> diff --git a/web/src/components/custom-form/checkpoint-input.tsx b/web/src/components/custom-form/checkpoint-input.tsx new file mode 100644 index 0000000..6b9f08a --- /dev/null +++ b/web/src/components/custom-form/checkpoint-input.tsx @@ -0,0 +1,89 @@ +import type { AutoFormInputComponentProps } from "../ui/auto-form/types"; +import { FormControl, FormItem, FormLabel } from "../ui/form"; +import { LoadingIcon } from "@/components/LoadingIcon"; +import * as React from "react"; +import AutoFormInput from "../ui/auto-form/fields/input"; +import { useDebouncedCallback } from "use-debounce"; +import { CivitaiModel } from "./ModelPickerView"; +import { z } from "zod"; +import { insertCivitaiCheckpointSchema } from "@/db/schema"; + +function getUrl(civitai_url: string) { + // expect to be a URL to be https://civitai.com/models/36520 + // possiblity with slugged name and query-param modelVersionId + + const baseUrl = "https://civitai.com/api/v1/models/"; + const url = new URL(civitai_url); + const pathSegments = url.pathname.split("/"); + const modelId = pathSegments[pathSegments.indexOf("models") + 1]; + const modelVersionId = url.searchParams.get("modelVersionId"); + + return { url: baseUrl + modelId, modelVersionId }; +} + +export default function AutoFormCheckpointInput( + props: AutoFormInputComponentProps, +) { + const [loading, setLoading] = React.useState(false); + const [modelRes, setModelRes] = React.useState< + z.infer + >(); + const [modelVersionid, setModelVersionId] = React.useState(); + const { label, isRequired, fieldProps, zodItem, fieldConfigItem } = props; + + const handleSearch = useDebouncedCallback((search) => { + const validationResult = insertCivitaiCheckpointSchema.shape.civitai_url + .safeParse(search); + if (!validationResult.success) { + console.error(validationResult.error); + // Optionally set an error state here + return; + } + + setLoading(true); + + const controller = new AbortController(); + const { url, modelVersionId: versionId } = getUrl(search); + setModelVersionId(versionId); + fetch(url, { + signal: controller.signal, + }) + .then((x) => x.json()) + .then((a) => { + const res = CivitaiModel.parse(a); + console.log(a); + console.log(res); + setModelRes(res); + setLoading(false); + }); + + return () => { + controller.abort(); + setLoading(false); + }; + }, 300); + + const modifiedField = { + ...fieldProps, + // onChange: (event: React.ChangeEvent) => { + // handleSearch(event.target.value); + // }, + }; + + return ( + + {fieldConfigItem.inputProps?.showLabel && ( + + {label} + {isRequired && *} + + )} + + + + + ); +} diff --git a/web/src/components/ui/auto-form/config.ts b/web/src/components/ui/auto-form/config.ts index 4fdb9ea..113ffbd 100644 --- a/web/src/components/ui/auto-form/config.ts +++ b/web/src/components/ui/auto-form/config.ts @@ -8,6 +8,7 @@ import AutoFormSwitch from "./fields/switch"; import AutoFormTextarea from "./fields/textarea"; import AutoFormModelsPicker from "@/components/custom-form/model-picker"; import AutoFormSnapshotPicker from "@/components/custom-form/snapshot-picker"; +import AutoFormCheckpointInput from "@/components/custom-form/checkpoint-input"; export const INPUT_COMPONENTS = { checkbox: AutoFormCheckbox, @@ -22,6 +23,7 @@ export const INPUT_COMPONENTS = { // Customs snapshot: AutoFormSnapshotPicker, models: AutoFormModelsPicker, + checkpoints: AutoFormCheckpointInput, }; /** diff --git a/web/src/db/schema.ts b/web/src/db/schema.ts index f3e7216..ad55897 100644 --- a/web/src/db/schema.ts +++ b/web/src/db/schema.ts @@ -1,13 +1,13 @@ -import { relations, type InferSelectModel } from "drizzle-orm"; +import { type InferSelectModel, relations } from "drizzle-orm"; import { - text, - pgSchema, - uuid, + boolean, integer, - timestamp, jsonb, pgEnum, - boolean, + pgSchema, + text, + timestamp, + uuid, } from "drizzle-orm/pg-core"; import { createInsertSchema } from "drizzle-zod"; import { z } from "zod"; @@ -87,7 +87,7 @@ export const workflowVersionRelations = relations( fields: [workflowVersionTable.workflow_id], references: [workflowTable.id], }), - }) + }), ); export const workflowRunStatus = pgEnum("workflow_run_status", [ @@ -136,10 +136,11 @@ export const workflowRunsTable = dbSchema.table("workflow_runs", { () => workflowVersionTable.id, { onDelete: "set null", - } + }, ), - workflow_inputs: - jsonb("workflow_inputs").$type>(), + workflow_inputs: jsonb("workflow_inputs").$type< + Record + >(), workflow_id: uuid("workflow_id") .notNull() .references(() => workflowTable.id, { @@ -171,7 +172,7 @@ export const workflowRunRelations = relations( fields: [workflowRunsTable.workflow_id], references: [workflowTable.id], }), - }) + }), ); // We still want to keep the workflow run record. @@ -195,7 +196,7 @@ export const workflowOutputRelations = relations( fields: [workflowRunOutputs.run_id], references: [workflowRunsTable.id], }), - }) + }), ); // when user delete, also delete all the workflow versions @@ -228,7 +229,7 @@ export const snapshotType = z.object({ z.object({ hash: z.string(), disabled: z.boolean(), - }) + }), ), file_custom_nodes: z.array(z.any()), }); @@ -243,7 +244,7 @@ export const showcaseMedia = z.array( z.object({ url: z.string(), isCover: z.boolean().default(false), - }) + }), ); export const showcaseMediaNullable = z @@ -251,7 +252,7 @@ export const showcaseMediaNullable = z z.object({ url: z.string(), isCover: z.boolean().default(false), - }) + }), ) .nullable(); @@ -275,8 +276,9 @@ export const deploymentsTable = dbSchema.table("deployments", { .notNull() .references(() => machinesTable.id), description: text("description"), - showcase_media: - jsonb("showcase_media").$type>(), + showcase_media: jsonb("showcase_media").$type< + z.infer + >(), environment: deploymentEnvironment("environment").notNull(), created_at: timestamp("created_at").defaultNow().notNull(), updated_at: timestamp("updated_at").defaultNow().notNull(), @@ -329,121 +331,291 @@ export const apiKeyTable = dbSchema.table("api_keys", { updated_at: timestamp("updated_at").defaultNow().notNull(), }); -const civitaiModelVersion = z.object({ - id: z.number(), - modelId: z.number(), - name: z.string(), - createdAt: z.string(), - updatedAt: z.string(), - status: z.string(), - publishedAt: z.string(), - trainedWords: z.array(z.string()).optional(), - trainingStatus: z.string().optional(), - trainingDetails: z.string().optional(), - baseModel: z.string(), - baseModelType: z.string(), - earlyAccessTimeFrame: z.number(), - description: z.string().optional(), - vaeId: z.string().optional(), - stats: z.object({ - downloadCount: z.number(), - ratingCount: z.number(), - rating: z.number() - }), - files: z.array(z.object({ - id: z.number(), - sizeKB: z.number(), - name: z.string(), - type: z.string(), - metadata: z.object({ - fp: z.string(), - size: z.string(), - format: z.string() - }), - pickleScanResult: z.string(), - pickleScanMessage: z.string().optional(), - virusScanResult: z.string(), - virusScanMessage: z.string().optional(), - scannedAt: z.string(), - hashes: z.object({ - AutoV1: z.string(), - AutoV2: z.string(), - SHA256: z.string(), - CRC32: z.string(), - BLAKE3: z.string(), - AutoV3: z.string() - }), - downloadUrl: z.string(), - primary: z.boolean() - })), - images: z.array(z.object({ - url: z.string(), - nsfw: z.string(), - width: z.number(), - height: z.number(), - hash: z.string(), - type: z.string(), - metadata: z.object({ - hash: z.string(), - size: z.number(), - width: z.number(), - height: z.number() - }), - meta: z.any() - })), - downloadUrl: z.string() -}); +// const civitaiModelVersion = z.object({ +// id: z.number(), +// modelId: z.number(), +// name: z.string(), +// createdAt: z.string(), +// updatedAt: z.string(), +// status: z.string(), +// publishedAt: z.string(), +// trainedWords: z.array(z.string()).optional(), +// trainingStatus: z.string().optional(), +// trainingDetails: z.string().optional(), +// baseModel: z.string(), +// baseModelType: z.string(), +// earlyAccessTimeFrame: z.number(), +// description: z.string().optional(), +// vaeId: z.string().optional(), +// stats: z.object({ +// downloadCount: z.number(), +// ratingCount: z.number(), +// rating: z.number(), +// }), +// files: z.array(z.object({ +// id: z.number(), +// sizeKB: z.number(), +// name: z.string(), +// type: z.string(), +// metadata: z.object({ +// fp: z.string(), +// size: z.string(), +// format: z.string(), +// }), +// pickleScanResult: z.string(), +// pickleScanMessage: z.string().optional(), +// virusScanResult: z.string(), +// virusScanMessage: z.string().optional(), +// scannedAt: z.string(), +// hashes: z.object({ +// AutoV1: z.string(), +// AutoV2: z.string(), +// SHA256: z.string(), +// CRC32: z.string(), +// BLAKE3: z.string(), +// AutoV3: z.string(), +// }), +// downloadUrl: z.string(), +// primary: z.boolean(), +// })), +// images: z.array(z.object({ +// url: z.string(), +// nsfw: z.string(), +// width: z.number(), +// height: z.number(), +// hash: z.string(), +// type: z.string(), +// metadata: z.object({ +// hash: z.string(), +// size: z.number(), +// width: z.number(), +// height: z.number(), +// }), +// meta: z.any(), +// })), +// downloadUrl: z.string(), +// }); +// +// const civitaiModelResponseType = z.object({ +// id: z.number(), +// name: z.string(), +// description: z.string().optional(), +// type: z.string(), +// poi: z.boolean(), +// nsfw: z.boolean(), +// allowNoCredit: z.boolean(), +// allowCommercialUse: z.string(), +// allowDerivatives: z.boolean(), +// allowDifferentLicense: z.boolean(), +// stats: z.object({ +// downloadCount: z.number(), +// favoriteCount: z.number(), +// commentCount: z.number(), +// ratingCount: z.number(), +// rating: z.number(), +// tippedAmountCount: z.number(), +// }), +// creator: z.object({ +// username: z.string(), +// image: z.string(), +// }), +// tags: z.array(z.string()), +// modelVersions: z.array(civitaiModelVersion), +// }); -const civitaiModelResponseType = z.object({ +export const CivitaiModel = z.object({ id: z.number(), name: z.string(), - description: z.string().optional(), + description: z.string(), type: z.string(), - poi: z.boolean(), - nsfw: z.boolean(), - allowNoCredit: z.boolean(), - allowCommercialUse: z.string(), - allowDerivatives: z.boolean(), - allowDifferentLicense: z.boolean(), - stats: z.object({ - downloadCount: z.number(), - favoriteCount: z.number(), - commentCount: z.number(), - ratingCount: z.number(), - rating: z.number(), - tippedAmountCount: z.number() - }), - creator: z.object({ - username: z.string(), - image: z.string() - }), + // poi: z.boolean(), + // nsfw: z.boolean(), + // allowNoCredit: z.boolean(), + // allowCommercialUse: z.string(), + // allowDerivatives: z.boolean(), + // allowDifferentLicense: z.boolean(), + // stats: z.object({ + // downloadCount: z.number(), + // favoriteCount: z.number(), + // commentCount: z.number(), + // ratingCount: z.number(), + // rating: z.number(), + // tippedAmountCount: z.number(), + // }), + creator: z + .object({ + username: z.string().nullable(), + image: z.string().nullable().default(null), + }) + .nullable(), tags: z.array(z.string()), - modelVersions: z.array(civitaiModelVersion) + modelVersions: z.array( + z.object({ + id: z.number(), + modelId: z.number(), + name: z.string(), + createdAt: z.string(), + updatedAt: z.string(), + status: z.string(), + publishedAt: z.string(), + trainedWords: z.array(z.unknown()), + trainingStatus: z.string().nullable(), + trainingDetails: z.string().nullable(), + baseModel: z.string(), + baseModelType: z.string().nullable(), + earlyAccessTimeFrame: z.number(), + description: z.string().nullable(), + vaeId: z.number().nullable(), + stats: z.object({ + downloadCount: z.number(), + ratingCount: z.number(), + rating: z.number(), + }), + files: z.array( + z.object({ + id: z.number(), + sizeKB: z.number(), + name: z.string(), + type: z.string(), + // metadata: z.object({ + // fp: z.string().nullable().optional(), + // size: z.string().nullable().optional(), + // format: z.string().nullable().optional(), + // }), + // pickleScanResult: z.string(), + // pickleScanMessage: z.string(), + // virusScanResult: z.string(), + // virusScanMessage: z.string().nullable(), + // scannedAt: z.string(), + // hashes: z.object({ + // AutoV1: z.string().nullable().optional(), + // AutoV2: z.string().nullable().optional(), + // SHA256: z.string().nullable().optional(), + // CRC32: z.string().nullable().optional(), + // BLAKE3: z.string().nullable().optional(), + // }), + downloadUrl: z.string(), + // primary: z.boolean().default(false), + }), + ), + images: z.array( + z.object({ + id: z.number(), + url: z.string(), + nsfw: z.string(), + width: z.number(), + height: z.number(), + hash: z.string(), + type: z.string(), + metadata: z.object({ + hash: z.string(), + width: z.number(), + height: z.number(), + }), + meta: z.any(), + }), + ), + downloadUrl: z.string(), + }), + ), }); +export const resourceUpload = pgEnum("resource_upload", [ + "started", + "failed", + "succeded", +]); -export const checkpoints = dbSchema.table("checkpoints", { +export const modelUploadType = pgEnum("model_upload_type", [ + "civitai", + "huggingface", + "other", +]); + +export const checkpointTable = dbSchema.table("checkpoints", { id: uuid("id").primaryKey().defaultRandom().notNull(), user_id: text("user_id") - .references(() => usersTable.id, { - // onDelete: "cascade", - }), // if null it's global? + .references(() => usersTable.id, {}), // perhaps a "special" user_id for global checkpoints org_id: text("org_id"), description: text("description"), - civitai_id : text('civitai_id'), - civitai_url : text('civitai_url'), - civitai_details: jsonb("civitai_model_response").$type>(), + checkpoint_volume_id: uuid("checkpoint_volume_id") + .notNull() + .references(() => workflowRunsTable.id, { + onDelete: "cascade", + }).notNull(), - hf_url: text('hf_url'), - s3_url: text('s3_url'), + model_name: text("model_name"), + + civitai_id: text("civitai_id"), + civitai_version_id: text("civitai_version_id"), + civitai_url: text("civitai_url"), + civitai_download_url: text("civitai_download_url"), + civitai_model_response: jsonb("civitai_model_response").$type< + z.infer + >(), + + hf_url: text("hf_url"), + s3_url: text("s3_url"), + user_url: text("client_url"), + + is_public: boolean("is_public").notNull().default(false), + status: resourceUpload("status").notNull().default("started"), + upload_machine_id: text("upload_machine_id"), + upload_type: modelUploadType("upload_type").notNull(), created_at: timestamp("created_at").defaultNow().notNull(), updated_at: timestamp("updated_at").defaultNow().notNull(), }); +export const insertCivitaiCheckpointSchema = createInsertSchema( + checkpointTable, + { + civitai_url: (schema) => + schema.civitai_url.trim().url({ message: "URL required" }).includes( + "civitai.com/models", + { message: "civitai.com/models link required" }, + ), + }, +); + +export const checkpointVolumeTable = dbSchema.table("checkpointVolumeTable", { + id: uuid("id").primaryKey().defaultRandom().notNull(), + user_id: text("user_id") + .references(() => usersTable.id, { + // onDelete: "cascade", + }), + org_id: text("org_id"), + volume_name: text("volume_name").notNull(), + created_at: timestamp("created_at").defaultNow().notNull(), + updated_at: timestamp("updated_at").defaultNow().notNull(), + disabled: boolean("disabled").default(false).notNull(), +}); + +export const checkpointRelations = relations(checkpointTable, ({ one }) => ({ + user: one(usersTable, { + fields: [checkpointTable.user_id], + references: [usersTable.id], + }), + volume: one(checkpointVolumeTable, { + fields: [checkpointTable.checkpoint_volume_id], + references: [checkpointVolumeTable.id], + }), +})); + +export const checkpointVolumeRelations = relations( + checkpointVolumeTable, + ({ many }) => ({ + checkpoint: many(checkpointTable), + }), +); + export type UserType = InferSelectModel; export type WorkflowType = InferSelectModel; export type MachineType = InferSelectModel; export type WorkflowVersionType = InferSelectModel; export type DeploymentType = InferSelectModel; +export type CheckpointType = InferSelectModel; +export type CheckpointVolumeType = InferSelectModel< + typeof checkpointVolumeTable +>; diff --git a/web/src/server/addCheckpointSchema.tsx b/web/src/server/addCheckpointSchema.tsx new file mode 100644 index 0000000..b83f5a3 --- /dev/null +++ b/web/src/server/addCheckpointSchema.tsx @@ -0,0 +1,5 @@ +import { insertCivitaiCheckpointSchema } from "@/db/schema"; + +export const addCivitaiCheckpointSchema = insertCivitaiCheckpointSchema.pick({ + civitai_url: true, +}); diff --git a/web/src/server/curdCheckpoint.ts b/web/src/server/curdCheckpoint.ts new file mode 100644 index 0000000..4173ccc --- /dev/null +++ b/web/src/server/curdCheckpoint.ts @@ -0,0 +1,222 @@ +"use server"; + +import { auth } from "@clerk/nextjs"; +import { + checkpointTable, + CheckpointType, + checkpointVolumeTable, + CheckpointVolumeType, + CivitaiModel, +} from "@/db/schema"; +import { withServerPromise } from "./withServerPromise"; +import { redirect } from "next/navigation"; +import { db } from "@/db/db"; +import type { z } from "zod"; +import { headers } from "next/headers"; +import { addCivitaiCheckpointSchema } from "./addCheckpointSchema"; +import { and, eq, isNull } from "drizzle-orm"; + +export async function getCheckpoints() { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + const checkpoints = await db + .select() + .from(checkpointTable) + .where( + orgId + ? eq(checkpointTable.org_id, orgId) + // make sure org_id is null + : and( + eq(checkpointTable.user_id, userId), + isNull(checkpointTable.org_id), + ), + ); + return checkpoints; +} + +export async function getCheckpointById(id: string) { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + const checkpoint = await db + .select() + .from(checkpointTable) + .where( + and( + orgId ? eq(checkpointTable.org_id, orgId) : and( + eq(checkpointTable.user_id, userId), + isNull(checkpointTable.org_id), + ), + eq(checkpointTable.id, id), + ), + ); + return checkpoint[0]; +} + +export async function getCheckpointVolumes() { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + const checkpointVolume = await db + .select() + .from(checkpointVolumeTable) + .where( + and( + orgId + ? eq(checkpointVolumeTable.org_id, orgId) + // make sure org_id is null + : and( + eq(checkpointVolumeTable.user_id, userId), + isNull(checkpointVolumeTable.org_id), + ), + eq(checkpointVolumeTable.disabled, false), + ), + ); + return checkpointVolume; +} + +export async function addCheckpointVolume() { + const { userId, orgId } = auth(); + if (!userId) throw new Error("No user id"); + + // Insert the new volume into the checkpointVolumeTable + const insertedVolume = await db + .insert(checkpointVolumeTable) + .values({ + user_id: userId, + org_id: orgId, + volume_name: `checkpoints_${userId}`, + // created_at and updated_at will be set to current timestamp by default + disabled: false, // Default value + }) + .returning(); // Returns the inserted row + + return insertedVolume; +} + +function getUrl(civitai_url: string) { + // expect to be a URL to be https://civitai.com/models/36520 + // possiblity with slugged name and query-param modelVersionId + const baseUrl = "https://civitai.com/api/v1/models/"; + const url = new URL(civitai_url); + const pathSegments = url.pathname.split("/"); + const modelId = pathSegments[pathSegments.indexOf("models") + 1]; + const modelVersionId = url.searchParams.get("modelVersionId"); + + return { url: baseUrl + modelId, modelVersionId }; +} + +export const addCivitaiCheckpoint = withServerPromise( + async (data: z.infer) => { + const { userId, orgId } = auth(); + + if (!data.civitai_url) return { error: "no civitai_url" }; + if (!userId) return { error: "No user id" }; + + const { url, modelVersionId } = getUrl(data?.civitai_url); + const civitaiModelRes = await fetch(url) + .then((x) => x.json()) + .then((a) => { + console.log(a) + return CivitaiModel.parse(a); + }); + + if (civitaiModelRes.modelVersions?.length === 0) { + return; // no versions to download + } + + let selectedModelVersion; + let selectedModelVersionId: string | null = modelVersionId; + if (!selectedModelVersionId) { + selectedModelVersion = civitaiModelRes.modelVersions[0]; + selectedModelVersionId = civitaiModelRes.modelVersions[0].id.toString(); + } else { + selectedModelVersion = civitaiModelRes.modelVersions.find((version) => + version.id.toString() === selectedModelVersionId + ); + if (!selectedModelVersion) { + return; // version id is wrong + } + selectedModelVersionId = selectedModelVersion?.id.toString(); + } + + const checkpointVolumes = await getCheckpointVolumes(); + let cVolume; + if (checkpointVolumes.length === 0) { + const volume = await addCheckpointVolume(); + cVolume = volume[0]; + } else { + cVolume = checkpointVolumes[0]; + } + + const a = await db + .insert(checkpointTable) + .values({ + user_id: userId, + org_id: orgId, + upload_type: "civitai", + civitai_id: civitaiModelRes.id.toString(), + civitai_version_id: selectedModelVersionId, + civitai_model_response: civitaiModelRes, + checkpoint_volume_id: cVolume.id, + }) + .returning(); + + const b = a[0]; + + await uploadCheckpoint(data, b, cVolume); + redirect(`/checkpoints/${b.id}`); + }, +); + +async function uploadCheckpoint( + data: z.infer, + b: CheckpointType, + v: CheckpointVolumeType, +) { + const headersList = headers(); + + const domain = headersList.get("x-forwarded-host") || ""; + const protocol = headersList.get("x-forwarded-proto") || ""; + + if (domain === "") { + throw new Error("No domain"); + } + + // Call remote builder + const result = await fetch( + `${process.env.MODAL_BUILDER_URL!}/upload_volume`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + download_url: data.civitai_url, + volume_name: v.volume_name, + callback_url: `${protocol}://${domain}/api/volume-updated`, + }), + }, + ); + + if (!result.ok) { + // const error_log = await result.text(); + // await db + // .update(checkpointTable) + // .set({ + // ...data, + // status: "error", + // build_log: error_log, + // }) + // .where(eq(machinesTable.id, b.id)); + // throw new Error(`Error: ${result.statusText} ${error_log}`); + } else { + // setting the build machine id + const json = await result.json(); + await db + .update(checkpointTable) + .set({ + ...data, + // build_machine_instance_id: json.build_machine_instance_id, + }) + .where(eq(machinesTable.id, b.id)); + } +} diff --git a/web/src/server/getAllUserCheckpoints.tsx b/web/src/server/getAllUserCheckpoints.tsx new file mode 100644 index 0000000..8e3df1b --- /dev/null +++ b/web/src/server/getAllUserCheckpoints.tsx @@ -0,0 +1,39 @@ +import { db } from "@/db/db"; +import { + checkpointTable, +} from "@/db/schema"; +import { auth } from "@clerk/nextjs"; +import { and, desc, eq, isNull } from "drizzle-orm"; + +export async function getAllUserCheckpoints() { + const { userId, orgId } = await auth(); + + if (!userId) { + return null; + } + + const checkpoints = await db.query.checkpointTable.findMany({ + with: { + user: { + columns: { + name: true, + }, + }, + }, + columns: { + id: true, + updated_at: true, + name: true, + civitai_url: true, + civitai_model_response: true, + is_public: true, + }, + orderBy: desc(checkpointTable.updated_at), + where: + orgId != undefined + ? eq(checkpointTable.org_id, orgId) + : and(eq(checkpointTable.user_id, userId), isNull(checkpointTable.org_id)), + }); + + return checkpoints; +} diff --git a/web/src/types/civitai.ts b/web/src/types/civitai.ts new file mode 100644 index 0000000..8c2cf6f --- /dev/null +++ b/web/src/types/civitai.ts @@ -0,0 +1,55 @@ +import { z } from 'zod'; + +export const creatorSchema = z.object({ + username: z.string(), + image: z.string(), +}); + +export const statsSchema = z.object({ + downloadCount: z.number(), + favoriteCount: z.number(), + commentCount: z.number(), + ratingCount: z.number(), + rating: z.number(), + tippedAmountCount: z.number(), +}); + +export const modelVersionSchema = z.object({ + id: z.number(), + modelId: z.number(), + name: z.string(), + createdAt: z.string(), + updatedAt: z.string(), + status: z.string(), + publishedAt: z.string(), + trainedWords: z.array(z.any()), // Replace with more specific type if known + trainingStatus: z.any().optional(), + trainingDetails: z.any().optional(), + baseModel: z.string(), + baseModelType: z.string(), + earlyAccessTimeFrame: z.number(), + description: z.string().optional(), + vaeId: z.any().optional(), // Replace with more specific type if known + stats: statsSchema.optional(), // If stats structure is known, replace with specific type + files: z.array(z.any()), // Replace with more specific type if known + images: z.array(z.any()), // Replace with more specific type if known + downloadUrl: z.string(), +}); + +export const CivitaiModel = z.object({ + id: z.number(), + name: z.string(), + description: z.string(), + type: z.string(), + poi: z.boolean(), + nsfw: z.boolean(), + allowNoCredit: z.boolean(), + allowCommercialUse: z.string(), + allowDerivatives: z.boolean(), + allowDifferentLicense: z.boolean(), + stats: statsSchema, + creator: creatorSchema, + tags: z.array(z.string()), + modelVersions: z.array(modelVersionSchema), +}); +