Skip to content

Commit

Permalink
Deprecate conda-based images; improve micromamba support
Browse files Browse the repository at this point in the history
  • Loading branch information
@mwaskom
mwaskom committed May 2, 2024
1 parent 3372f7a commit c6a7890
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 43 deletions.
67 changes: 47 additions & 20 deletions modal/image.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ class _Image(_Object, type_prefix="im"):
"""Base class for container images to run functions in.
Do not construct this class directly; instead use one of its static factory methods,
such as `modal.Image.debian_slim`, `modal.Image.from_registry`, or `modal.Image.conda`.
such as `modal.Image.debian_slim`, `modal.Image.from_registry`, or `modal.Image.micromamba`.
"""

force_build: bool
Expand Down Expand Up @@ -869,9 +869,16 @@ def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
@staticmethod
def conda(python_version: Optional[str] = None, force_build: bool = False) -> "_Image":
"""
A Conda base image, using miniconda3 and derived from the official Docker Hub image.
In most cases, using [`Image.micromamba()`](/docs/reference/modal.Image#micromamba) with [`micromamba_install`](/docs/reference/modal.Image#micromamba_install) is recommended over `Image.conda()`, as it leads to significantly faster image build times.
DEPRECATED A Conda base image, using miniconda3.
This constructor has been deprecated in favor of [`Image.micromamba()`](/docs/reference/modal.Image#micromamba),
which can be used with [`micromamba_install`](/docs/reference/modal.Image#micromamba_install).
Images will build faster and more reliably with `micromamba`.
"""
msg = (
"The `Image.conda` constructor has deprecated in favor of the faster and more reliable `Image.micromamba`."
)
deprecation_warning((2024, 5, 2), msg)

def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
nonlocal python_version
Expand Down Expand Up @@ -948,8 +955,15 @@ def conda_install(
secrets: Sequence[_Secret] = [],
gpu: GPU_T = None,
) -> "_Image":
"""Install a list of additional packages using Conda. Note that in most cases, using [`Image.micromamba()`](/docs/reference/modal.Image#micromamba) with [`micromamba_install`](/docs/reference/modal.Image#micromamba_install)
is recommended over `conda_install`, as it leads to significantly faster image build times."""
"""DEPRECATED Install additional packages using Conda.
This method has been deprecated in favor of [`micromamba_install`](/docs/reference/modal.Image#micromamba_install),
which should be used with the [`Image.micromamba()`](/docs/reference/modal.Image#micromamba) constructor.
Images will build faster and more reliably with `micromamba`.
"""

msg = "The `Image.conda_install` method has deprecated in favor of the faster and more reliable `Image.micromamba_install`."
deprecation_warning((2024, 5, 2), msg)

pkgs = _flatten_str_args("conda_install", "packages", packages)
if not pkgs:
Expand Down Expand Up @@ -982,7 +996,16 @@ def conda_update_from_environment(
secrets: Sequence[_Secret] = [],
gpu: GPU_T = None,
) -> "_Image":
"""Update a Conda environment using dependencies from a given environment.yml file."""
"""DEPRECATED Update a Conda environment using dependencies from a given environment.yml file.
This method has been deprecated in favor of the faster and more reliable `Image.micromamba_install`
method and its `spec_file` parameter.
"""
msg = (
"The `Image.conda_install_update_from_environment` method has deprecated in favor of the faster"
" and more reliable `Image.micromamba_install` and its `spec_file` parameter."
)
deprecation_warning((2024, 5, 2), msg)

def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
context_files = {"/environment.yml": os.path.expanduser(environment_yml)}
Expand All @@ -1008,10 +1031,7 @@ def micromamba(
python_version: Optional[str] = None,
force_build: bool = False, # Ignore cached builds, similar to 'docker build --no-cache'
) -> "_Image":
"""
A Micromamba base image. Micromamba allows for fast building of small Conda-based containers.
In most cases it will be faster than using [`Image.conda()`](/docs/reference/modal.Image#conda).
"""
"""A Micromamba base image. Micromamba allows for fast building of small Conda-based containers."""

def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
nonlocal python_version
Expand Down Expand Up @@ -1040,27 +1060,36 @@ def micromamba_install(
self,
# A list of Python packages, eg. ["numpy", "matplotlib>=3.5.0"]
*packages: Union[str, List[str]],
# A list of Conda channels, eg. ["conda-forge", "nvidia"]
# A local path to a file containing package specifications
spec_file: Optional[str] = None,
# A list of Conda channels, eg. ["conda-forge", "nvidia"]. Uses "conda-forge" when not specified.
channels: List[str] = [],
force_build: bool = False, # Ignore cached builds, similar to 'docker build --no-cache'
secrets: Sequence[_Secret] = [],
gpu: GPU_T = None,
) -> "_Image":
"""Install a list of additional packages using micromamba."""

pkgs = _flatten_str_args("micromamba_install", "packages", packages)
if not pkgs:
if not pkgs and spec_file is None:
return self
channels = channels or ["conda-forge"]

def build_dockerfile(version: ImageBuilderVersion) -> DockerfileSpec:
package_args = " ".join(shlex.quote(pkg) for pkg in pkgs)
channel_args = "".join(f" -c {channel}" for channel in channels)

space = " " if package_args else ""
remote_spec_file = "" if spec_file is None else f"/{os.path.basename(spec_file)}"
file_arg = "" if spec_file is None else f"{space}-f {remote_spec_file}"
copy_commands = [] if spec_file is None else [f"COPY {remote_spec_file} {remote_spec_file}"]

commands = [
"FROM base",
f"RUN micromamba install {package_args}{channel_args} --yes",
*copy_commands,
f"RUN micromamba install {package_args}{file_arg}{channel_args} --yes",
]
return DockerfileSpec(commands=commands, context_files={})
context_files = {} if spec_file is None else {remote_spec_file: os.path.expanduser(spec_file)}
return DockerfileSpec(commands=commands, context_files=context_files)

return _Image._from_args(
base_images={"base": self},
Expand Down Expand Up @@ -1462,16 +1491,14 @@ def my_build_function():
)

def env(self, vars: Dict[str, str]) -> "_Image":
"""Sets the environmental variables of the image.
"""Sets the environment variables in an Image.
**Example**
```python
image = (
modal.Image.conda()
.env({"CONDA_OVERRIDE_CUDA": "11.2"})
.conda_install("jax", "cuda-nvcc", channels=["conda-forge", "nvidia"])
.pip_install("dm-haiku", "optax")
modal.Image.debian_slim()
.env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
)
```
"""
Expand Down
75 changes: 52 additions & 23 deletions test/image_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,6 @@ def test_image_base(builder_version, servicer, client, test_dir):
(Image.debian_slim, ()),
(Image.from_registry, ("ubuntu",)),
(Image.from_dockerfile, (test_dir / "supports" / "test-dockerfile",)),
(Image.conda, ()),
(Image.micromamba, ()),
]
for meth, args in constructors:
Expand Down Expand Up @@ -133,13 +132,12 @@ def test_python_version(builder_version, servicer, client, python_version):
commands = get_all_dockerfile_commands(app.image.object_id, servicer)
assert re.match(rf"FROM python:{expected_dockerhub_python}-slim-{expected_dockerhub_debian}", commands)

for constructor in [Image.conda, Image.micromamba]:
app.image = constructor() if python_version is None else constructor(python_version)
if python_version is None and builder_version == "2023.12":
expected_python = "3.9"
with app.run(client):
commands = get_all_dockerfile_commands(app.image.object_id, servicer)
assert re.search(rf"install.* python={expected_python}", commands)
app.image = Image.micromamba() if python_version is None else Image.micromamba(python_version)
if python_version is None and builder_version == "2023.12":
expected_python = "3.9"
with app.run(client):
commands = get_all_dockerfile_commands(app.image.object_id, servicer)
assert re.search(rf"install.* python={expected_python}", commands)


def test_image_python_packages(builder_version, servicer, client):
Expand Down Expand Up @@ -216,7 +214,7 @@ def test_empty_install(builder_version, servicer, client):
.pip_install([], [], [], [])
.apt_install([])
.run_commands()
.conda_install()
.micromamba_install()
)

with app.run(client=client):
Expand Down Expand Up @@ -307,8 +305,21 @@ def test_image_pip_install_private_repos(builder_version, servicer, client):
)


def test_dockerfile_image(builder_version, servicer, client):
path = os.path.join(os.path.dirname(__file__), "supports/test-dockerfile")

app = App(image=Image.from_dockerfile(path))

with app.run(client=client):
layers = get_image_layers(app.image.object_id, servicer)

assert any("RUN pip install numpy" in cmd for cmd in layers[1].dockerfile_commands)


def test_conda_install(builder_version, servicer, client):
app = App(image=Image.conda().pip_install("numpy").conda_install("pymc3", "theano").pip_install("scikit-learn"))
with pytest.warns(DeprecationError, match="Image.micromamba"):
image = Image.conda().pip_install("numpy").conda_install("pymc3", "theano").pip_install("scikit-learn")
app = App(image=image)

with app.run(client=client):
layers = get_image_layers(app.image.object_id, servicer)
Expand All @@ -318,26 +329,40 @@ def test_conda_install(builder_version, servicer, client):
assert any("pip install numpy" in cmd for cmd in layers[2].dockerfile_commands)


def test_dockerfile_image(builder_version, servicer, client):
path = os.path.join(os.path.dirname(__file__), "supports/test-dockerfile")
def test_conda_update_from_environment(builder_version, servicer, client):
path = os.path.join(os.path.dirname(__file__), "supports/test-conda-environment.yml")

app = App(image=Image.from_dockerfile(path))
with pytest.warns(DeprecationError, match="Image.micromamba"):
app = App(image=Image.conda().conda_update_from_environment(path))

with app.run(client=client):
layers = get_image_layers(app.image.object_id, servicer)

assert any("RUN pip install numpy" in cmd for cmd in layers[1].dockerfile_commands)

assert any("RUN conda env update" in cmd for cmd in layers[0].dockerfile_commands)
assert any(b"foo=1.0" in f.data for f in layers[0].context_files)
assert any(b"bar=2.1" in f.data for f in layers[0].context_files)

def test_conda_update_from_environment(builder_version, servicer, client):
path = os.path.join(os.path.dirname(__file__), "supports/test-conda-environment.yml")

app = App(image=Image.conda().conda_update_from_environment(path))
def test_micromamba_install(builder_version, servicer, client):
spec_file = os.path.join(os.path.dirname(__file__), "supports/test-conda-environment.yml")
image = (
Image.micromamba()
.pip_install("numpy")
.micromamba_install("pymc3", "theano")
.pip_install("scikit-learn")
.micromamba_install(spec_file=spec_file)
)
app = App(image=image)

with app.run(client=client):
layers = get_image_layers(app.image.object_id, servicer)

assert any("RUN conda env update" in cmd for cmd in layers[0].dockerfile_commands)
print(layers[0].dockerfile_commands)
assert any("COPY /test-conda-environment.yml" in cmd for cmd in layers[0].dockerfile_commands)
assert any("micromamba install -f /test-conda-environment.yml" in cmd for cmd in layers[0].dockerfile_commands)
assert any("pip install scikit-learn" in cmd for cmd in layers[1].dockerfile_commands)
assert any("micromamba install pymc3 theano --yes" in cmd for cmd in layers[2].dockerfile_commands)
assert any("pip install numpy" in cmd for cmd in layers[3].dockerfile_commands)
assert any(b"foo=1.0" in f.data for f in layers[0].context_files)
assert any(b"bar=2.1" in f.data for f in layers[0].context_files)

Expand Down Expand Up @@ -776,7 +801,8 @@ def get_hash(img: Image) -> str:
img = Image.from_registry("ubuntu:22.04")
assert get_hash(img) == "b5f1cc544a412d1b23a5ebf9a8859ea9a86975ecbc7325b83defc0ce3fe956d3"

img = Image.conda()
with pytest.warns(DeprecationError):
img = Image.conda()
assert get_hash(img) == "f69d6af66fb5f1a2372a61836e6166ce79ebe2cd628d12addea8e8e80cc98dc1"

img = Image.micromamba()
Expand All @@ -785,7 +811,8 @@ def get_hash(img: Image) -> str:
img = Image.from_dockerfile(test_dir / "supports" / "test-dockerfile")
assert get_hash(img) == "0aec2f66f28ee7511c1b36604214ae7b40d9bc1fa3e6b8883001e933a966ff78"

img = Image.conda(python_version="3.12")
with pytest.warns(DeprecationError):
img = Image.conda(python_version="3.12")
assert get_hash(img) == "c4b3f7350116d323dded29c9c9b78b62593f0fc943ccf83a09b27185bfdc2a07"

img = Image.micromamba(python_version="3.12")
Expand All @@ -799,13 +826,15 @@ def get_hash(img: Image) -> str:
img = base.pip_install("torch~=2.2", "transformers==4.23.0", pre=True, index_url="agi.se")
assert get_hash(img) == "2a4fa8e3b32c70a41b3a3efd5416540b1953430543f6c27c984e7f969c2ca874"

img = base.conda_install("torch=2.2", "transformers<4.23.0", channels=["conda-forge", "my-channel"])
with pytest.warns(DeprecationError):
img = base.conda_install("torch=2.2", "transformers<4.23.0", channels=["conda-forge", "my-channel"])
assert get_hash(img) == "dd6f27f636293996a64a98c250161d8092cb23d02629d9070493f00aad8d7266"

img = base.pip_install_from_requirements(test_dir / "supports" / "test-requirements.txt")
assert get_hash(img) == "69d41e699d4ecef399e51e8460f8857aa0ec57f71f00eca81c8886ec062e5c2b"

img = base.conda_update_from_environment(test_dir / "supports" / "test-conda-environment.yml")
with pytest.warns(DeprecationError):
img = base.conda_update_from_environment(test_dir / "supports" / "test-conda-environment.yml")
assert get_hash(img) == "00940e0ee2998bfe0a337f51a5fdf5f4b29bf9d42dda3635641d44bfeb42537e"

img = base.poetry_install_from_file(
Expand Down

0 comments on commit c6a7890

Please sign in to comment.