blah

builder/modal-builder/src/volume-builder/app.py

@@ -0,0 +1,69 @@
+import modal
+from config import config
+import os
+import subprocess
+
+stub = modal.Stub()
+
+# Volume names may only contain alphanumeric characters, dashes, periods, and underscores, and must be less than 64 characters in length.
+def is_valid_name(name: str) -> bool:
+    allowed_characters = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._")
+    return 0 < len(name) <= 64 and all(char in allowed_characters for char in name)
+
+def create_volumes(volume_names, paths):
+    path_to_vol = {}
+    for volume_name in volume_names.keys():
+        if not is_valid_name(volume_name):
+            pass
+        modal_volume = modal.Volume.persisted(volume_name)
+        path_to_vol[paths[volume_name]] = modal_volume
+ 
+    return path_to_vol
+
+vol_name_to_links = config["volume_names"]
+vol_name_to_path = config["volume_paths"]
+callback_url = config["callback_url"]
+callback_body = config["callback_body"]
+
+volumes = create_volumes(vol_name_to_links, vol_name_to_path)
+image = ( 
+   modal.Image.debian_slim().apt_install("wget").pip_install("requests")
+)
+
+print(vol_name_to_links)
+print(vol_name_to_path)
+print(volumes)
+
+# download config { "download_url": "", "folder_path": ""}
+timeout=5000
+@stub.function(volumes=volumes, image=image, timeout=timeout, gpu=None)
+def download_model(volume_name, download_config):
+    import requests
+    download_url = download_config["download_url"]
+    folder_path = download_config["folder_path"]
+    volume_base_path = vol_name_to_path[volume_name]
+    model_store_path = os.path.join(volume_base_path, folder_path)
+
+    subprocess.run(["wget", download_url, "--content-disposition", "-P", model_store_path])
+    subprocess.run(["ls", "-la", volume_base_path])
+    subprocess.run(["ls", "-la", model_store_path])
+    volumes[volume_base_path].commit()
+
+    status =  {"status": "success"}
+    requests.post(callback_url, json={**status, **callback_body})
+
+
+@stub.local_entrypoint()
+def simple_download():
+    import requests
+    print(vol_name_to_links)
+    print([(vol_name, link) for vol_name,link in vol_name_to_links.items()])
+    try:
+        list(download_model.starmap([(vol_name, link) for vol_name,link in vol_name_to_links.items()]))
+    except modal.exception.FunctionTimeoutError as e:
+        status =  {"status": "failed", "error_logs": f"{str(e)}", "timeout": timeout}
+        requests.post(callback_url, json={**status, **callback_body})
+    except Exception as e:
+        status =  {"status": "failed", "error_logs": str(e)}
+        requests.post(callback_url, json={**status, **callback_body})
+        

builder/modal-builder/src/volume-builder/config.py

@@ -0,0 +1,17 @@
+config = {
+    "volume_names": {
+        "test": {
+            "download_url": "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose (1).png",
+            "folder_path": "images"
+        }
+    }, 
+    "volume_paths": {
+        "test": "/volumes/something"
+    },
+    "callback_url": "",
+    "callback_body": {
+        "checkpoint_id": "",
+        "volume_id": "",
+        "folder_path": "images",
+    }
+}