work

builder/modal-builder/src/main.py

@@ -224,6 +224,20 @@ async def websocket_endpoint(websocket: WebSocket, machine_id: str):
 #     return {"Hello": "World"}
 
 
+class UploadBody(BaseModel):
+    download_url: str
+    volume_name: str
+    # callback_url: str
+
+@app.post("/upload_volume")
+async def upload_checkpoint(body: UploadBody):
+    download_url = body.download_url
+    volume_name = body.download_url
+    # callback_url = body.callback_url
+    # check that thi
+    return
+
+
 @app.post("/create")
 async def create_machine(item: Item):
     global last_activity_time

builder/modal-builder/src/template/data/insert_models.py

@@ -3,9 +3,9 @@ This is a standalone script to download models into a modal Volume using civitai
 
 Example Usage
 `modal run insert_models::insert_model --civitai-url https://civitai.com/models/36520/ghostmix`
-This inserts an individual model from a civitai url (public not API url)
+This inserts an individual model from a civitai url 
 
-`modal run insert_models::insert_models` 
+`modal run insert_models::insert_models_civitai_api` 
 This inserts a bunch of models based on the models retrieved by civitai
 
 civitai's API reference https://github.com/civitai/civitai/wiki/REST-API-Reference
@@ -13,27 +13,24 @@ civitai's API reference https://github.com/civitai/civitai/wiki/REST-API-Referen
 import modal
 import subprocess
 import requests
+import json
 
 stub = modal.Stub()
 
 # NOTE: volume name can be variable
-volume = modal.Volume.persisted("private-model-store")
+volume = modal.Volume.persisted("rah")
 model_store_path = "/vol/models"
 MODEL_ROUTE = "models"
 image = (
     modal.Image.debian_slim().apt_install("wget").pip_install("requests")
 )
 
-@stub.function(volumes={model_store_path: volume}, gpu="any", image=image, timeout=600)
-def download_model(model):
-    # wget https://civitai.com/api/download/models/{modelVersionId} --content-disposition
-    # model_id = model['modelVersions'][0]['id']
-    # download_url = f"https://civitai.com/api/download/models/{model_id}"
-
-    download_url = model['modelVersions'][0]['downloadUrl'] 
+@stub.function(volumes={model_store_path: volume}, image=image, timeout=50000, gpu=None)
+def download_model(download_url):
+    print(download_url)
     subprocess.run(["wget", download_url, "--content-disposition", "-P", model_store_path])
     subprocess.run(["ls", "-la", model_store_path])
-    volume.commit()  
+    volume.commit()
 
 # file is raw output from Civitai API https://github.com/civitai/civitai/wiki/REST-API-Reference
 
@@ -52,40 +49,53 @@ def get_civitai_models(model_type: str, sort: str = "Highest Rated", page: int =
 @stub.function()
 def get_civitai_model_url(civitai_url: str):
     # Validate the URL
-    if not civitai_url.startswith("https://civitai.com/models/"):
+
+    if civitai_url.startswith("https://civitai.com/api/"):
+        api_url = civitai_url
+    elif civitai_url.startswith("https://civitai.com/models/"):  
+        try:
+            model_id = civitai_url.split("/")[4]
+            int(model_id) 
+        except (IndexError, ValueError):
+            return None 
+        api_url = f"https://civitai.com/api/v1/models/{model_id}"
+    else:
         return "Error: URL must be from civitai.com and contain /models/"
 
-    # Extract the model ID
-    try:
-        model_id = civitai_url.split("/")[4]
-        int(model_id)  # Check if the ID is an integer
-    except (IndexError, ValueError):
-        return None #Error: Invalid model ID in URL
-
-    # Make the API request
-    api_url = f"https://civitai.com/api/v1/models/{model_id}"
     response = requests.get(api_url)
-
     # Check for successful response
     if response.status_code != 200:
         return f"Error: Unable to fetch data from {api_url}"
-
     # Return the response data
     return response.json()
 
 
 
 @stub.local_entrypoint()
-def insert_models(type: str = "Checkpoint", sort = "Highest Rated", page: int = 1):
+def insert_models_civitai_api(type: str = "Checkpoint", sort = "Highest Rated", page: int = 1):
     civitai_models = get_civitai_models.local(type, sort, page)
     if civitai_models:
-        for _ in download_model.map(civitai_models['items'][1:]):
+        for _ in download_model.map(map(lambda model: model['modelVersions'][0]['downloadUrl'], civitai_models['items'])):
             pass
     else:
         print("Failed to retrieve models.")
 
 @stub.local_entrypoint()
 def insert_model(civitai_url: str):
-    civitai_model = get_civitai_model_url.local(civitai_url)
-    if civitai_model: 
-        download_model.remote(civitai_model)
+    if civitai_url.startswith("'https://civitai.com/api/download/models/"):
+        download_url = civitai_url
+    else:
+        civitai_model = get_civitai_model_url.local(civitai_url)
+        if civitai_model:
+            download_url = civitai_model['modelVersions'][0]['downloadUrl']
+        else:
+            return "invalid URL"
+
+    download_model.remote(download_url)
+
+@stub.local_entrypoint()
+def simple_download():
+    download_urls = ['https://civitai.com/api/download/models/119057', 'https://civitai.com/api/download/models/130090', 'https://civitai.com/api/download/models/31859', 'https://civitai.com/api/download/models/128713', 'https://civitai.com/api/download/models/179657', 'https://civitai.com/api/download/models/143906', 'https://civitai.com/api/download/models/9208', 'https://civitai.com/api/download/models/136078', 'https://civitai.com/api/download/models/134065', 'https://civitai.com/api/download/models/288775', 'https://civitai.com/api/download/models/95263', 'https://civitai.com/api/download/models/288982', 'https://civitai.com/api/download/models/87153', 'https://civitai.com/api/download/models/10638', 'https://civitai.com/api/download/models/263809', 'https://civitai.com/api/download/models/130072', 'https://civitai.com/api/download/models/117019', 'https://civitai.com/api/download/models/95256', 'https://civitai.com/api/download/models/197181', 'https://civitai.com/api/download/models/256915', 'https://civitai.com/api/download/models/118945', 'https://civitai.com/api/download/models/125843', 'https://civitai.com/api/download/models/179015', 'https://civitai.com/api/download/models/245598', 'https://civitai.com/api/download/models/223670', 'https://civitai.com/api/download/models/90072', 'https://civitai.com/api/download/models/290817', 'https://civitai.com/api/download/models/154097', 'https://civitai.com/api/download/models/143497', 'https://civitai.com/api/download/models/5637']
+
+    for _ in download_model.map(download_urls):
+        pass

builder/modal-builder/src/volume-builder/app.py

@@ -0,0 +1,54 @@
+import modal
+from config import config
+import os
+import uuid
+import subprocess
+
+stub = modal.Stub()
+
+base_path = "/volumes"
+
+# Volume names may only contain alphanumeric characters, dashes, periods, and underscores, and must be less than 64 characters in length.
+def is_valid_name(name: str) -> bool:
+    allowed_characters = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._")
+    return 0 < len(name) <= 64 and all(char in allowed_characters for char in name)
+
+def create_volumes(volume_names):
+    path_to_vol = {}
+    vol_to_path = {}
+    for volume_name in volume_names.keys():
+        if not is_valid_name(volume_name):
+            pass
+        modal_volume = modal.Volume.persisted(volume_name)
+        volume_path = create_volume_path(base_path)
+        path_to_vol[volume_path] = modal_volume
+        vol_to_path[volume_name] = volume_path
+ 
+    return (path_to_vol, vol_to_path)
+
+def create_volume_path(base_path: str):
+    random_path = str(uuid.uuid4())
+    return os.path.join(base_path, random_path)
+
+vol_name_to_links = config["volume_names"]
+(path_to_vol, vol_name_to_path) = create_volumes(vol_name_to_links)
+image = ( 
+   modal.Image.debian_slim().apt_install("wget").pip_install("requests")
+)
+
+print(vol_name_to_links)
+print(path_to_vol)
+print(vol_name_to_path)
+
+@stub.function(volumes=path_to_vol, image=image, timeout=5000, gpu=None)
+def download_model(volume_name, download_url):
+    model_store_path = vol_name_to_path[volume_name]
+    subprocess.run(["wget", download_url, "--content-disposition", "-P", model_store_path])
+    subprocess.run(["ls", "-la", model_store_path])
+    path_to_vol[model_store_path].commit()
+
+@stub.local_entrypoint()
+def simple_download():
+    print(vol_name_to_links)
+    print([(vol_name, link) for vol_name,link in vol_name_to_links.items()])
+    list(download_model.starmap([(vol_name, link) for vol_name,link in vol_name_to_links.items()]))

builder/modal-builder/src/volume-builder/config.py

@@ -0,0 +1,5 @@
+config = {
+    "volume_names": {
+        "eg1": "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose (1).png"
+    }, 
+}