Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding mac / windows specific tests for standalone command. #169

Merged
merged 15 commits into from
Aug 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/run-on-gpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@ on:
paths:
- "comfy_cli/**"
- "!comfy_cli/test_**"
- "!.github/**"
pull_request:
branches:
- main
paths:
- "comfy_cli/**"
- "!comfy_cli/test_**"
- "!.github/**"

jobs:
test-cli-gpu:
Expand Down
32 changes: 32 additions & 0 deletions .github/workflows/test-mac.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: "Mac Specific Commands"
on:
pull_request:
branches:
- main

jobs:
test:
runs-on: macos-latest
env:
PYTHONIOENCODING: "utf8"

steps:
- name: Check out code
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12

- name: Install Dependencies
run: |
python -m venv venv
source venv/bin/activate
python -m pip install --upgrade pip
pip install pytest
pip install -e .
comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --m-series --skip-manager
comfy --workspace ./ComfyUI standalone --platform macos
comfy standalone --rehydrate
./python/bin/python ComfyUI/main.py --cpu --quick-test-for-ci
34 changes: 34 additions & 0 deletions .github/workflows/test-windows.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: "Windows Specific Commands"
on:
pull_request:
branches:
- main

jobs:
test:
runs-on: windows-latest
env:
PYTHONIOENCODING: "utf8"

steps:
- name: Check out code
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12

- name: Install Dependencies
run: |
python -m venv venv
.\venv\Scripts\Activate.ps1
Get-Command python
python -m pip install --upgrade pip
pip install pytest
pip install -e .
comfy --skip-prompt --workspace ./ComfyUI install --fast-deps --nvidia --cuda-version 12.1 --skip-manager
comfy --workspace ./ComfyUI standalone --platform windows --proc x86_64
ls
comfy standalone --rehydrate --platform windows --proc x86_64
./python/python.exe ComfyUI/main.py --cpu --quick-test-for-ci
77 changes: 16 additions & 61 deletions comfy_cli/standalone.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
import platform as os_platform
import shutil
import subprocess
import tarfile
from pathlib import Path
from typing import Optional

import requests
from rich.live import Live
from rich.progress import Progress, TextColumn
from rich.table import Table

from comfy_cli.constants import OS, PROC
from comfy_cli.typing import PathLike
from comfy_cli.utils import download_progress, get_os, get_proc
from comfy_cli.utils import create_tarball, download_url, extract_tarball, get_os, get_proc
from comfy_cli.uv import DependencyCompiler

_here = Path(__file__).expanduser().resolve().parent
Expand All @@ -37,6 +32,7 @@ def download_standalone_python(
tag: str = "latest",
flavor: str = "install_only",
cwd: PathLike = ".",
show_progress: bool = True,
) -> PathLike:
"""grab a pre-built distro from the python-build-standalone project. See
https://gregoryszorc.com/docs/python-build-standalone/main/"""
Expand All @@ -59,9 +55,9 @@ def download_standalone_python(

name = f"cpython-{version}+{tag}-{target}-{flavor}"
fname = f"{name}.tar.gz"
url = f"{asset_url_prefix}/{fname}"
url = f"{asset_url_prefix.rstrip('/')}/{fname.lstrip('/')}"

return download_progress(url, fname, cwd=cwd)
return download_url(url, fname, cwd=cwd, show_progress=show_progress)


class StandalonePython:
Expand All @@ -74,43 +70,32 @@ def FromDistro(
flavor: str = "install_only",
cwd: PathLike = ".",
name: PathLike = "python",
):
show_progress: bool = True,
) -> "StandalonePython":
fpath = download_standalone_python(
platform=platform,
proc=proc,
version=version,
tag=tag,
flavor=flavor,
cwd=cwd,
show_progress=show_progress,
)
return StandalonePython.FromTarball(fpath, name)

@staticmethod
def FromTarball(fpath: PathLike, name: PathLike = "python") -> "StandalonePython":
def FromTarball(fpath: PathLike, name: PathLike = "python", show_progress: bool = True) -> "StandalonePython":
fpath = Path(fpath)

with tarfile.open(fpath) as tar:
info = tar.next()
old_name = info.name.split("/")[0]

old_rpath = fpath.parent / old_name
rpath = fpath.parent / name

# clean the tar file expand target and the final target
shutil.rmtree(old_rpath, ignore_errors=True)
shutil.rmtree(rpath, ignore_errors=True)

with tarfile.open(fpath) as tar:
tar.extractall()

shutil.move(old_rpath, rpath)
extract_tarball(inPath=fpath, outPath=rpath, show_progress=show_progress)

return StandalonePython(rpath=rpath)

def __init__(self, rpath: PathLike):
self.rpath = Path(rpath)
self.name = self.rpath.name
if os_platform.system() == "Windows":
if get_os() == OS.WINDOWS:
self.bin = self.rpath
self.executable = self.bin / "python.exe"
else:
Expand Down Expand Up @@ -170,48 +155,18 @@ def dehydrate_comfy_deps(
extraSpecs=extraSpecs,
)
self.dep_comp.compile_deps()
self.dep_comp.fetch_dep_wheels()

skip_uv = get_os() == OS.WINDOWS
self.dep_comp.fetch_dep_wheels(skip_uv=skip_uv)

def rehydrate_comfy_deps(self):
self.dep_comp = DependencyCompiler(
executable=self.executable, outDir=self.rpath, reqFilesCore=[], reqFilesExt=[]
)
self.dep_comp.install_wheels_directly()

def to_tarball(self, outPath: Optional[PathLike] = None, progress: bool = True):
outPath = self.rpath.with_suffix(".tgz") if outPath is None else Path(outPath)

# do a little clean up prep
outPath.unlink(missing_ok=True)
def to_tarball(self, outPath: Optional[PathLike] = None, show_progress: bool = True):
# remove any __pycache__ before creating archive
self.clean()

if progress:
fileSize = sum(f.stat().st_size for f in self.rpath.glob("**/*"))

barProg = Progress()
addTar = barProg.add_task("[cyan]Creating tarball...", total=fileSize)
pathProg = Progress(TextColumn("{task.description}"))
pathTar = pathProg.add_task("")

progress_table = Table.grid()
progress_table.add_row(barProg)
progress_table.add_row(pathProg)

_size = 0

def _filter(tinfo: tarfile.TarInfo):
nonlocal _size
pathProg.update(pathTar, description=tinfo.path)
barProg.advance(addTar, _size)
_size = Path(tinfo.path).stat().st_size
return tinfo
else:
_filter = None

with Live(progress_table, refresh_per_second=10):
with tarfile.open(outPath, "w:gz") as tar:
tar.add(self.rpath.relative_to(Path(".").expanduser().resolve()), filter=_filter)

if progress:
barProg.advance(addTar, _size)
pathProg.update(pathTar, description="")
create_tarball(inPath=self.rpath, outPath=outPath, show_progress=show_progress)
Loading
Loading