rest

builder/modal-builder/src/main.py

@@ -8,6 +8,7 @@ from enum import Enum
 import json
 import subprocess
 import time
+from uuid import uuid4
 from contextlib import asynccontextmanager
 import asyncio
 import threading
@@ -19,6 +20,7 @@ from urllib.parse import parse_qs
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.types import ASGIApp, Scope, Receive, Send
 
+
 from concurrent.futures import ThreadPoolExecutor
 
 # executor = ThreadPoolExecutor(max_workers=5)
@@ -227,15 +229,47 @@ async def websocket_endpoint(websocket: WebSocket, machine_id: str):
 class UploadBody(BaseModel):
     download_url: str
     volume_name: str
+    volume_id: str
     # callback_url: str
 
 @app.post("/upload_volume")
 async def upload_checkpoint(body: UploadBody):
+    global last_activity_time
+    last_activity_time = time.time()
+    logger.info(f"Extended inactivity time to {global_timeout}")
+
     download_url = body.download_url
-    volume_name = body.download_url
+    volume_name = body.volume_name
     # callback_url = body.callback_url
+    
+    folder_path = f"/app/builds/{body.volume_id}"
+
+    cp_process = await asyncio.subprocess.create_subprocess_exec("cp", "-r", "/app/src/volume-builder", folder_path)
+    await cp_process.wait()
+
+    # Write the config file
+    config = {
+        "volume_names": {
+            volume_name: download_url
+        }, 
+        "paths": {
+            volume_name: f'/volumes/{uuid4()}'
+        }, 
+    }
+
+    await asyncio.subprocess.create_subprocess_shell(
+        f"modal run app.py",
+        # stdout=asyncio.subprocess.PIPE,
+        # stderr=asyncio.subprocess.PIPE,
+        cwd=folder_path,
+        env={**os.environ, "COLUMNS": "10000"}
+    )
+
+    with open(f"{folder_path}/config.py", "w") as f:
+        f.write("config = " + json.dumps(config))
+
     # check that thi
-    return
+    return JSONResponse(status_code=200, content={"message": "Volume uploading", "build_machine_instance_id": fly_instance_id})
 
 
 @app.post("/create")

builder/modal-builder/src/volume-builder/app.py

@@ -1,51 +1,42 @@
 import modal
 from config import config
 import os
-import uuid
 import subprocess
 
 stub = modal.Stub()
 
-base_path = "/volumes"
-
 # Volume names may only contain alphanumeric characters, dashes, periods, and underscores, and must be less than 64 characters in length.
 def is_valid_name(name: str) -> bool:
     allowed_characters = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._")
     return 0 < len(name) <= 64 and all(char in allowed_characters for char in name)
 
-def create_volumes(volume_names):
+def create_volumes(volume_names, paths):
     path_to_vol = {}
-    vol_to_path = {}
     for volume_name in volume_names.keys():
         if not is_valid_name(volume_name):
             pass
         modal_volume = modal.Volume.persisted(volume_name)
-        volume_path = create_volume_path(base_path)
-        path_to_vol[volume_path] = modal_volume
-        vol_to_path[volume_name] = volume_path
+        path_to_vol[paths[volume_name]] = modal_volume
  
-    return (path_to_vol, vol_to_path)
-
-def create_volume_path(base_path: str):
-    random_path = str(uuid.uuid4())
-    return os.path.join(base_path, random_path)
+    return path_to_vol
 
 vol_name_to_links = config["volume_names"]
-(path_to_vol, vol_name_to_path) = create_volumes(vol_name_to_links)
+vol_name_to_path = config["paths"]
+volumes = create_volumes(vol_name_to_links, vol_name_to_path)
 image = ( 
    modal.Image.debian_slim().apt_install("wget").pip_install("requests")
 )
 
 print(vol_name_to_links)
-print(path_to_vol)
 print(vol_name_to_path)
+print(volumes)
 
-@stub.function(volumes=path_to_vol, image=image, timeout=5000, gpu=None)
+@stub.function(volumes=volumes, image=image, timeout=5000, gpu=None)
 def download_model(volume_name, download_url):
     model_store_path = vol_name_to_path[volume_name]
     subprocess.run(["wget", download_url, "--content-disposition", "-P", model_store_path])
     subprocess.run(["ls", "-la", model_store_path])
-    path_to_vol[model_store_path].commit()
+    volumes[model_store_path].commit()
 
 @stub.local_entrypoint()
 def simple_download():

builder/modal-builder/src/volume-builder/config.py

@@ -1,5 +1,8 @@
 config = {
     "volume_names": {
-        "eg1": "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose (1).png"
+        "test": "https://pub-6230db03dc3a4861a9c3e55145ceda44.r2.dev/openpose-pose (1).png"
     }, 
+    "paths": {
+        "test": "/volumes/something"
+    }
 }