From 3035a5976e9a9f24fb6d069b933f8a118cb97f06 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 28 Aug 2024 17:26:08 -0700 Subject: [PATCH 01/15] Add test for install and standalone on windows. --- .github/workflows/test-windows.yml | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/test-windows.yml diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml new file mode 100644 index 0000000..721cd85 --- /dev/null +++ b/.github/workflows/test-windows.yml @@ -0,0 +1,31 @@ +name: "Windows Specific Commands" +on: + pull_request: + branches: + - main + +jobs: + test: + name: "Run Tests on Multiple Platforms" + runs-on: windows-latest + env: + PYTHONIOENCODING: "utf8" + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install -e . + comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --nvidia --cuda-version 12.1 + comfy --workspace ./ComfyUI standalone --platform windows --proc x86_64 + ls + comfy standalone --rehydrate --platform windows --proc x86_64 From 6d4f43cc55c2fff523c2d2fed0c2f2e8305203be Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 28 Aug 2024 17:50:31 -0700 Subject: [PATCH 02/15] Test in venv. --- .github/workflows/test-mac.yml | 31 ++++++++++++++++++++++++++++++ .github/workflows/test-windows.yml | 3 ++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-mac.yml diff --git a/.github/workflows/test-mac.yml b/.github/workflows/test-mac.yml new file mode 100644 index 0000000..f12a547 --- /dev/null +++ b/.github/workflows/test-mac.yml @@ -0,0 +1,31 @@ +name: "Mac Specific Commands" +on: + pull_request: + branches: + - main + +jobs: + test: + runs-on: macos-latest + env: + PYTHONIOENCODING: "utf8" + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Dependencies + run: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + pip install pytest + pip install -e . + comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --m-series + comfy --workspace ./ComfyUI standalone --platform macos + comfy standalone --rehydrate diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 721cd85..6d62d8e 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -6,7 +6,6 @@ on: jobs: test: - name: "Run Tests on Multiple Platforms" runs-on: windows-latest env: PYTHONIOENCODING: "utf8" @@ -22,6 +21,8 @@ jobs: - name: Install Dependencies run: | + python -m venv venv + venv\Scripts\activate python -m pip install --upgrade pip pip install pytest pip install -e . From 1ed22b9f6aacaba2128a4501949ebbce098376bd Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 28 Aug 2024 17:59:17 -0700 Subject: [PATCH 03/15] Skip Manager. --- .github/workflows/test-mac.yml | 3 ++- .github/workflows/test-windows.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mac.yml b/.github/workflows/test-mac.yml index f12a547..c9f4800 100644 --- a/.github/workflows/test-mac.yml +++ b/.github/workflows/test-mac.yml @@ -26,6 +26,7 @@ jobs: python -m pip install --upgrade pip pip install pytest pip install -e . - comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --m-series + comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --m-series --skip-manager comfy --workspace ./ComfyUI standalone --platform macos comfy standalone --rehydrate + ./python/bin/python ComfyUI/main.py --cpu --quick-test-for-ci diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 6d62d8e..50b6eba 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -26,7 +26,8 @@ jobs: python -m pip install --upgrade pip pip install pytest pip install -e . - comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --nvidia --cuda-version 12.1 + comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --nvidia --cuda-version 12.1 --skip-manager comfy --workspace ./ComfyUI standalone --platform windows --proc x86_64 ls comfy standalone --rehydrate --platform windows --proc x86_64 + ./python/python.exe ComfyUI/main.py --cpu --quick-test-for-ci From 88a2069a2162a3b9a10a70050384d2205f88a31e Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 28 Aug 2024 18:09:57 -0700 Subject: [PATCH 04/15] Fix activate command. --- .github/workflows/test-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 50b6eba..45f5449 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -22,7 +22,7 @@ jobs: - name: Install Dependencies run: | python -m venv venv - venv\Scripts\activate + .\venv\Scripts\Activate.ps1 python -m pip install --upgrade pip pip install pytest pip install -e . From d2c596ada8194cf8636afa3d5591302c073c8894 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 28 Aug 2024 18:42:51 -0700 Subject: [PATCH 05/15] Update. --- .github/workflows/run-on-gpu.yml | 2 -- .github/workflows/test-windows.yml | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run-on-gpu.yml b/.github/workflows/run-on-gpu.yml index fe35995..1c5f0a3 100644 --- a/.github/workflows/run-on-gpu.yml +++ b/.github/workflows/run-on-gpu.yml @@ -6,13 +6,11 @@ on: - main paths: - "comfy_cli/**" - - "!comfy_cli/test_**" pull_request: branches: - main paths: - "comfy_cli/**" - - "!comfy_cli/test_**" jobs: test-cli-gpu: diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 45f5449..66f0488 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -23,6 +23,7 @@ jobs: run: | python -m venv venv .\venv\Scripts\Activate.ps1 + Get-Command python python -m pip install --upgrade pip pip install pytest pip install -e . From a2b4a1c4cc34f3def653d654928be3a27273f8a8 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 28 Aug 2024 18:44:26 -0700 Subject: [PATCH 06/15] Update. --- .github/workflows/run-on-gpu.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run-on-gpu.yml b/.github/workflows/run-on-gpu.yml index 1c5f0a3..a87ec3c 100644 --- a/.github/workflows/run-on-gpu.yml +++ b/.github/workflows/run-on-gpu.yml @@ -6,11 +6,15 @@ on: - main paths: - "comfy_cli/**" + - "!comfy_cli/test_**" + - "!.github/**" pull_request: branches: - main paths: - "comfy_cli/**" + - "!comfy_cli/test_**" + - "!.github/**" jobs: test-cli-gpu: From c5767d6dfd52bededf7ebd8cd8c97b8e1601cfbc Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 19:45:23 -0400 Subject: [PATCH 07/15] improve platform system determination --- comfy_cli/standalone.py | 3 +-- comfy_cli/utils.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/comfy_cli/standalone.py b/comfy_cli/standalone.py index 414cf9d..d2b33f7 100644 --- a/comfy_cli/standalone.py +++ b/comfy_cli/standalone.py @@ -1,4 +1,3 @@ -import platform as os_platform import shutil import subprocess import tarfile @@ -110,7 +109,7 @@ def FromTarball(fpath: PathLike, name: PathLike = "python") -> "StandalonePython def __init__(self, rpath: PathLike): self.rpath = Path(rpath) self.name = self.rpath.name - if os_platform.system() == "Windows": + if get_os() == OS.WINDOWS: self.bin = self.rpath self.executable = self.bin / "python.exe" else: diff --git a/comfy_cli/utils.py b/comfy_cli/utils.py index 586f022..7dc9bcd 100644 --- a/comfy_cli/utils.py +++ b/comfy_cli/utils.py @@ -6,7 +6,6 @@ import platform import shutil import subprocess -import sys from pathlib import Path import psutil @@ -39,12 +38,16 @@ def get_instance(*args, **kwargs): def get_os(): - if sys.platform == "darwin": + platform_system = platform.system().lower() + + if platform_system == "darwin": return OS.MACOS - elif "win" in sys.platform: + elif platform_system == "windows": return OS.WINDOWS - - return OS.LINUX + elif platform_system == "linux": + return OS.LINUX + else: + raise ValueError(f"Running on unsupported os {platform.system()}") def get_proc(): From 1dd3f15614d5d25bf511276ee2c342846818f77f Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 19:52:19 -0400 Subject: [PATCH 08/15] `DependencyCompiler`: add `reqs` parameter to `.Download` and `.Wheel` methods --- comfy_cli/uv.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index f807b77..4a36d6e 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -234,11 +234,12 @@ def Sync( @staticmethod def Download( cwd: PathLike, - reqFile: list[PathLike], executable: PathLike = sys.executable, extraUrl: Optional[str] = None, noDeps: bool = False, out: Optional[PathLike] = None, + reqs: Optional[list[str]] = None, + reqFile: Optional[list[PathLike]] = None, ) -> subprocess.CompletedProcess[Any]: """For now, the `download` cmd has no uv support, so use pip""" cmd = [ @@ -248,12 +249,6 @@ def Download( "download", ] - if isinstance(reqFile, (str, Path)): - cmd.extend(["-r", str(reqFile)]) - elif isinstance(reqFile, list): - for rf in reqFile: - cmd.extend(["-r", str(rf)]) - if extraUrl is not None: cmd.extend(["--extra-index-url", extraUrl]) @@ -263,25 +258,32 @@ def Download( if out is not None: cmd.extend(["-d", str(out)]) + if reqs is not None: + cmd.extend(reqs) + + if reqFile is not None: + for rf in reqFile: + cmd.extend(["--requirement", rf]) + return _check_call(cmd, cwd) @staticmethod def Wheel( cwd: PathLike, - reqFile: list[PathLike], executable: PathLike = sys.executable, extraUrl: Optional[str] = None, noDeps: bool = False, out: Optional[PathLike] = None, + reqs: Optional[list[str]] = None, + reqFile: Optional[list[PathLike]] = None, ) -> subprocess.CompletedProcess[Any]: """For now, the `wheel` cmd has no uv support, so use pip""" - cmd = [str(executable), "-m", "pip", "wheel"] - - if isinstance(reqFile, (str, Path)): - cmd.extend(["-r", str(reqFile)]) - elif isinstance(reqFile, list): - for rf in reqFile: - cmd.extend(["-r", str(rf)]) + cmd = [ + str(executable), + "-m", + "pip", + "wheel", + ] if extraUrl is not None: cmd.extend(["--extra-index-url", extraUrl]) @@ -292,6 +294,13 @@ def Wheel( if out is not None: cmd.extend(["-w", str(out)]) + if reqs is not None: + cmd.extend(reqs) + + if reqFile is not None: + for rf in reqFile: + cmd.extend(["--requirement", rf]) + return _check_call(cmd, cwd) @staticmethod From dc87ac65a86a880dbd4611c6c424c3d02ab5443c Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 20:18:19 -0400 Subject: [PATCH 09/15] refactored tarball creation/extraction to use `create_tarball`/`extract_tarball` --- comfy_cli/standalone.py | 68 ++++------------------ comfy_cli/utils.py | 124 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 63 deletions(-) diff --git a/comfy_cli/standalone.py b/comfy_cli/standalone.py index d2b33f7..879fd06 100644 --- a/comfy_cli/standalone.py +++ b/comfy_cli/standalone.py @@ -1,17 +1,13 @@ import shutil import subprocess -import tarfile from pathlib import Path from typing import Optional import requests -from rich.live import Live -from rich.progress import Progress, TextColumn -from rich.table import Table from comfy_cli.constants import OS, PROC from comfy_cli.typing import PathLike -from comfy_cli.utils import download_progress, get_os, get_proc +from comfy_cli.utils import create_tarball, download_url, extract_tarball, get_os, get_proc from comfy_cli.uv import DependencyCompiler _here = Path(__file__).expanduser().resolve().parent @@ -36,6 +32,7 @@ def download_standalone_python( tag: str = "latest", flavor: str = "install_only", cwd: PathLike = ".", + show_progress: bool = True, ) -> PathLike: """grab a pre-built distro from the python-build-standalone project. See https://gregoryszorc.com/docs/python-build-standalone/main/""" @@ -60,7 +57,7 @@ def download_standalone_python( fname = f"{name}.tar.gz" url = f"{asset_url_prefix}/{fname}" - return download_progress(url, fname, cwd=cwd) + return download_url(url, fname, cwd=cwd, show_progress=show_progress) class StandalonePython: @@ -73,7 +70,8 @@ def FromDistro( flavor: str = "install_only", cwd: PathLike = ".", name: PathLike = "python", - ): + show_progress: bool = True, + ) -> "StandalonePython": fpath = download_standalone_python( platform=platform, proc=proc, @@ -81,28 +79,16 @@ def FromDistro( tag=tag, flavor=flavor, cwd=cwd, + show_progress=show_progress, ) return StandalonePython.FromTarball(fpath, name) @staticmethod - def FromTarball(fpath: PathLike, name: PathLike = "python") -> "StandalonePython": + def FromTarball(fpath: PathLike, name: PathLike = "python", show_progress: bool = True) -> "StandalonePython": fpath = Path(fpath) - - with tarfile.open(fpath) as tar: - info = tar.next() - old_name = info.name.split("/")[0] - - old_rpath = fpath.parent / old_name rpath = fpath.parent / name - # clean the tar file expand target and the final target - shutil.rmtree(old_rpath, ignore_errors=True) - shutil.rmtree(rpath, ignore_errors=True) - - with tarfile.open(fpath) as tar: - tar.extractall() - - shutil.move(old_rpath, rpath) + extract_tarball(inPath=fpath, outPath=rpath, show_progress=show_progress) return StandalonePython(rpath=rpath) @@ -177,40 +163,8 @@ def rehydrate_comfy_deps(self): ) self.dep_comp.install_wheels_directly() - def to_tarball(self, outPath: Optional[PathLike] = None, progress: bool = True): - outPath = self.rpath.with_suffix(".tgz") if outPath is None else Path(outPath) - - # do a little clean up prep - outPath.unlink(missing_ok=True) + def to_tarball(self, outPath: Optional[PathLike] = None, show_progress: bool = True): + # remove any __pycache__ before creating archive self.clean() - if progress: - fileSize = sum(f.stat().st_size for f in self.rpath.glob("**/*")) - - barProg = Progress() - addTar = barProg.add_task("[cyan]Creating tarball...", total=fileSize) - pathProg = Progress(TextColumn("{task.description}")) - pathTar = pathProg.add_task("") - - progress_table = Table.grid() - progress_table.add_row(barProg) - progress_table.add_row(pathProg) - - _size = 0 - - def _filter(tinfo: tarfile.TarInfo): - nonlocal _size - pathProg.update(pathTar, description=tinfo.path) - barProg.advance(addTar, _size) - _size = Path(tinfo.path).stat().st_size - return tinfo - else: - _filter = None - - with Live(progress_table, refresh_per_second=10): - with tarfile.open(outPath, "w:gz") as tar: - tar.add(self.rpath.relative_to(Path(".").expanduser().resolve()), filter=_filter) - - if progress: - barProg.advance(addTar, _size) - pathProg.update(pathTar, description="") + create_tarball(inPath=self.rpath, outPath=outPath, show_progress=show_progress) diff --git a/comfy_cli/utils.py b/comfy_cli/utils.py index 7dc9bcd..6a1e374 100644 --- a/comfy_cli/utils.py +++ b/comfy_cli/utils.py @@ -6,12 +6,16 @@ import platform import shutil import subprocess +import tarfile from pathlib import Path +from typing import Optional import psutil import requests import typer from rich import print, progress +from rich.live import Live +from rich.table import Table from comfy_cli.constants import DEFAULT_COMFY_WORKSPACE, OS, PROC from comfy_cli.typing import PathLike @@ -100,7 +104,13 @@ def f(incomplete: str) -> list[str]: return f -def download_progress(url: str, fname: PathLike, cwd: PathLike = ".", allow_redirects: bool = True) -> PathLike: +def download_url( + url: str, + fname: PathLike, + cwd: PathLike = ".", + allow_redirects: bool = True, + show_progress: bool = True, +) -> PathLike: """download url to local file fname and show a progress bar. See https://stackoverflow.com/q/37573483""" cwd = Path(cwd).expanduser().resolve() @@ -110,12 +120,114 @@ def download_progress(url: str, fname: PathLike, cwd: PathLike = ".", allow_redi if response.status_code != 200: response.raise_for_status() # Will only raise for 4xx codes, so... raise RuntimeError(f"Request to {url} returned status code {response.status_code}") - fsize = int(response.headers.get("Content-Length", 0)) - desc = "(Unknown total file size)" if fsize == 0 else "" response.raw.read = functools.partial(response.raw.read, decode_content=True) # Decompress if needed - with progress.wrap_file(response.raw, total=fsize, description=desc) as response_raw: - with fpath.open("wb") as f: - shutil.copyfileobj(response_raw, f) + with fpath.open("wb") as f: + if show_progress: + fsize = int(response.headers.get("Content-Length", 0)) + desc = f"downloading {fname}..." + ("(Unknown total file size)" if fsize == 0 else "") + + with progress.wrap_file(response.raw, total=fsize, description=desc) as response_raw: + shutil.copyfileobj(response_raw, f) + else: + shutil.copyfileobj(response.raw, f) return fpath + + +def extract_tarball( + inPath: PathLike, + outPath: Optional[PathLike] = None, + show_progress: bool = True, +): + inPath = Path(inPath).expanduser().resolve() + outPath = inPath.with_suffix("") if outPath is None else Path(outPath).expanduser().resolve() + + with tarfile.open(inPath) as tar: + info = tar.next() + old_name = info.name.split("/")[0] + # path to top-level of extraction result + extractPath = inPath.with_name(old_name) + + # clean both the extraction path and the final target path + shutil.rmtree(extractPath, ignore_errors=True) + shutil.rmtree(outPath, ignore_errors=True) + + if show_progress: + fileSize = inPath.stat().st_size + + barProg = progress.Progress() + barTask = barProg.add_task("[cyan]extracting tarball...", total=fileSize) + pathProg = progress.Progress(progress.TextColumn("{task.description}")) + pathTask = pathProg.add_task("") + + progress_table = Table.grid() + progress_table.add_row(barProg) + progress_table.add_row(pathProg) + + _size = 0 + + def _filter(tinfo: tarfile.TarInfo, _path: Optional[PathLike] = None): + nonlocal _size + pathProg.update(pathTask, description=tinfo.path) + barProg.advance(barTask, _size) + _size = tinfo.size + return tinfo + else: + _filter = None + + with Live(progress_table, refresh_per_second=10): + with tarfile.open(inPath) as tar: + tar.extractall(filter=_filter) + + if show_progress: + barProg.advance(barTask, _size) + pathProg.update(pathTask, description="") + + shutil.move(extractPath, outPath) + + +def create_tarball( + inPath: PathLike, + outPath: Optional[PathLike] = None, + cwd: Optional[PathLike] = None, + show_progress: bool = True, +): + cwd = Path("." if cwd is None else cwd).expanduser().resolve() + inPath = Path(inPath).expanduser().resolve() + outPath = inPath.with_suffix(".tgz") if outPath is None else Path(outPath).expanduser().resolve() + + # clean the archive target path + outPath.unlink(missing_ok=True) + + if show_progress: + fileSize = sum(f.stat().st_size for f in inPath.glob("**/*")) + + barProg = progress.Progress() + barTask = barProg.add_task("[cyan]creating tarball...", total=fileSize) + pathProg = progress.Progress(progress.TextColumn("{task.description}")) + pathTask = pathProg.add_task("") + + progress_table = Table.grid() + progress_table.add_row(barProg) + progress_table.add_row(pathProg) + + _size = 0 + + def _filter(tinfo: tarfile.TarInfo): + nonlocal _size + pathProg.update(pathTask, description=tinfo.path) + barProg.advance(barTask, _size) + _size = Path(tinfo.path).stat().st_size + return tinfo + else: + _filter = None + + with Live(progress_table, refresh_per_second=10): + with tarfile.open(outPath, "w:gz") as tar: + # don't include parent paths in archive + tar.add(inPath.relative_to(cwd), filter=_filter) + + if show_progress: + barProg.advance(barTask, _size) + pathProg.update(pathTask, description="") From c31259c52bedc28e1a2f00c128feecbc2ef69ce6 Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 20:52:54 -0400 Subject: [PATCH 10/15] skip uv wheel when dehydrating standalone python on windows --- comfy_cli/standalone.py | 4 +++- comfy_cli/uv.py | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/comfy_cli/standalone.py b/comfy_cli/standalone.py index 879fd06..61f0d3b 100644 --- a/comfy_cli/standalone.py +++ b/comfy_cli/standalone.py @@ -155,7 +155,9 @@ def dehydrate_comfy_deps( extraSpecs=extraSpecs, ) self.dep_comp.compile_deps() - self.dep_comp.fetch_dep_wheels() + + skip_uv = get_os() == OS.WINDOWS + self.dep_comp.fetch_dep_wheels(skip_uv=skip_uv) def rehydrate_comfy_deps(self): self.dep_comp = DependencyCompiler( diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 4a36d6e..445e973 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -496,22 +496,48 @@ def sync_core_plus_ext(self): extraUrl=self.gpuUrl, ) - def fetch_dep_dists(self): + def fetch_dep_dists(self, skip_uv: bool = False): + if skip_uv: + reqFile = None + with open(self.out) as f: + reqs = [ + line + for line in f.readlines() + if not line.strip().startswith("#") and not line.strip().startswith("uv==") + ] + else: + reqFile = [self.out] + reqs = None + DependencyCompiler.Download( cwd=self.cwd, - reqFile=[self.out], executable=self.executable, extraUrl=self.gpuUrl, noDeps=True, out=self.outDir / "dists", + reqFile=reqFile, + reqs=reqs, ) - def fetch_dep_wheels(self): + def fetch_dep_wheels(self, skip_uv: bool = False): + if skip_uv: + reqFile = None + with open(self.out) as f: + reqs = [ + line + for line in f.readlines() + if not line.strip().startswith("#") and not line.strip().startswith("uv==") + ] + else: + reqFile = [self.out] + reqs = None + DependencyCompiler.Wheel( cwd=self.cwd, - reqFile=[self.out], executable=self.executable, extraUrl=self.gpuUrl, noDeps=True, out=self.outDir / "wheels", + reqFile=reqFile, + reqs=reqs, ) From 71402d39749edf7b98c66e2eac90924706f62b93 Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 22:04:25 -0400 Subject: [PATCH 11/15] small fixup to joining python standalone download url --- comfy_cli/standalone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_cli/standalone.py b/comfy_cli/standalone.py index 61f0d3b..c40861e 100644 --- a/comfy_cli/standalone.py +++ b/comfy_cli/standalone.py @@ -55,7 +55,7 @@ def download_standalone_python( name = f"cpython-{version}+{tag}-{target}-{flavor}" fname = f"{name}.tar.gz" - url = f"{asset_url_prefix}/{fname}" + url = f"{asset_url_prefix.rstrip('/')}/{fname.lstrip('/')}" return download_url(url, fname, cwd=cwd, show_progress=show_progress) From f606eef2e07f3e3fda57ff2afe867d84150a7d8d Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 22:05:04 -0400 Subject: [PATCH 12/15] improve parsing of reqs from reqFile --- comfy_cli/uv.py | 56 ++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 445e973..343aff8 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -43,6 +43,26 @@ def parse_uv_compile_error(err: str) -> tuple[str, list[str]]: return reqName, cast(list[str], reqRe.findall(err)) +def parse_req_file(rf: PathLike, skips: Optional[list[str]] = None): + skips = [] if skips is None else skips + + reqs: list[str] = [] + opts: list[str] = [] + with open(rf) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + elif "==" in line and line.split("==")[0] in skips: + continue + elif line.startswith("--"): + opts.extend(line.split()) + else: + reqs.append(line) + + return opts + reqs + + class DependencyCompiler: rocmPytorchUrl = "https://download.pytorch.org/whl/rocm6.1" nvidiaPytorchUrl = "https://download.pytorch.org/whl/cu121" @@ -497,47 +517,31 @@ def sync_core_plus_ext(self): ) def fetch_dep_dists(self, skip_uv: bool = False): - if skip_uv: - reqFile = None - with open(self.out) as f: - reqs = [ - line - for line in f.readlines() - if not line.strip().startswith("#") and not line.strip().startswith("uv==") - ] - else: - reqFile = [self.out] - reqs = None + skips = ["uv"] if skip_uv else None + reqs = parse_req_file(self.out, skips=skips) + + extraUrl = None if "--extra-index-url" in reqs else self.gpuUrl DependencyCompiler.Download( cwd=self.cwd, executable=self.executable, - extraUrl=self.gpuUrl, + extraUrl=extraUrl, noDeps=True, out=self.outDir / "dists", - reqFile=reqFile, reqs=reqs, ) def fetch_dep_wheels(self, skip_uv: bool = False): - if skip_uv: - reqFile = None - with open(self.out) as f: - reqs = [ - line - for line in f.readlines() - if not line.strip().startswith("#") and not line.strip().startswith("uv==") - ] - else: - reqFile = [self.out] - reqs = None + skips = ["uv"] if skip_uv else None + reqs = parse_req_file(self.out, skips=skips) + + extraUrl = None if "--extra-index-url" in reqs else self.gpuUrl DependencyCompiler.Wheel( cwd=self.cwd, executable=self.executable, - extraUrl=self.gpuUrl, + extraUrl=extraUrl, noDeps=True, out=self.outDir / "wheels", - reqFile=reqFile, reqs=reqs, ) From 7cbf6ba31e1d0d80b059e4b6017310d305d1b404 Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 23:19:51 -0400 Subject: [PATCH 13/15] add `tarfile.data_filter` to all tar filters to address secruity audit --- comfy_cli/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/comfy_cli/utils.py b/comfy_cli/utils.py index 6a1e374..06dc022 100644 --- a/comfy_cli/utils.py +++ b/comfy_cli/utils.py @@ -167,12 +167,13 @@ def extract_tarball( _size = 0 - def _filter(tinfo: tarfile.TarInfo, _path: Optional[PathLike] = None): + def _filter(tinfo: tarfile.TarInfo, _path: PathLike): nonlocal _size pathProg.update(pathTask, description=tinfo.path) barProg.advance(barTask, _size) _size = tinfo.size - return tinfo + + return tarfile.data_filter(tinfo, _path) else: _filter = None @@ -214,12 +215,13 @@ def create_tarball( _size = 0 - def _filter(tinfo: tarfile.TarInfo): + def _filter(tinfo: tarfile.TarInfo, _path: PathLike): nonlocal _size pathProg.update(pathTask, description=tinfo.path) barProg.advance(barTask, _size) _size = Path(tinfo.path).stat().st_size - return tinfo + + return tarfile.data_filter(tinfo, _path) else: _filter = None From 9a2e0521ac4d5fd38a621162a5215c5ba03ae390 Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 29 Aug 2024 23:52:37 -0400 Subject: [PATCH 14/15] revert tar security fix, since `tarfile.data_filter` is busted in many python versions - see: https://github.com/python/cpython/issues/107845 --- comfy_cli/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/comfy_cli/utils.py b/comfy_cli/utils.py index 06dc022..cbd024f 100644 --- a/comfy_cli/utils.py +++ b/comfy_cli/utils.py @@ -173,7 +173,9 @@ def _filter(tinfo: tarfile.TarInfo, _path: PathLike): barProg.advance(barTask, _size) _size = tinfo.size - return tarfile.data_filter(tinfo, _path) + # TODO: ideally we'd use data_filter here, but it's busted: https://github.com/python/cpython/issues/107845 + # return tarfile.data_filter(tinfo, _path) + return tinfo else: _filter = None @@ -215,13 +217,13 @@ def create_tarball( _size = 0 - def _filter(tinfo: tarfile.TarInfo, _path: PathLike): + def _filter(tinfo: tarfile.TarInfo): nonlocal _size pathProg.update(pathTask, description=tinfo.path) barProg.advance(barTask, _size) _size = Path(tinfo.path).stat().st_size - return tarfile.data_filter(tinfo, _path) + return tinfo else: _filter = None From 54386d966261c7202589ad020a197b72b6bebd0d Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 30 Aug 2024 00:09:32 -0400 Subject: [PATCH 15/15] add numpy<2 override on windows --- comfy_cli/uv.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 343aff8..8bfcd85 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -7,8 +7,9 @@ from typing import Any, Optional, Union, cast from comfy_cli import ui -from comfy_cli.constants import GPU_OPTION +from comfy_cli.constants import GPU_OPTION, OS from comfy_cli.typing import PathLike +from comfy_cli.utils import get_os def _run(cmd: list[str], cwd: PathLike, check: bool = True) -> subprocess.CompletedProcess[Any]: @@ -72,6 +73,7 @@ class DependencyCompiler: # ensure usage of {gpu} version of pytorch --extra-index-url {gpuUrl} torch + torchaudio torchsde torchvision """ @@ -397,6 +399,11 @@ def make_override(self): f.write(DependencyCompiler.overrideGpu.format(gpu=self.gpu, gpuUrl=self.gpuUrl)) f.write("\n\n") + # TODO: remove numpy<2 override once torch is compatible with numpy>=2 + if get_os() == OS.WINDOWS: + f.write("numpy<2\n") + f.write("\n\n") + completed = DependencyCompiler.Compile( cwd=self.cwd, reqFiles=self.reqFilesCore,