diff --git a/setup.py b/setup.py index 2cc1d9d..6edc166 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ 'privileged = rocker.extensions:Privileged', 'pulse = rocker.extensions:PulseAudio', 'rmw = rocker.rmw_extension:RMW', + 'ros_ws = rocker.ros_ws:RosWs', 'ssh = rocker.ssh_extension:Ssh', 'ulimit = rocker.ulimit_extension:Ulimit', 'user = rocker.extensions:User', diff --git a/src/rocker/ros_ws.py b/src/rocker/ros_ws.py new file mode 100644 index 0000000..d9081bf --- /dev/null +++ b/src/rocker/ros_ws.py @@ -0,0 +1,223 @@ +import em +import copy +import pkgutil +import getpass +import os +from rocker.core import get_user_name +from rocker.extensions import RockerExtension +from rocker.volume_extension import Volume +import tempfile +from vcstools import VcsClient +import xml.etree.ElementTree as ET +import yaml + + +class RosWs(RockerExtension): + + name = "ros_ws" + + @classmethod + def get_name(cls): + return cls.name + + def __init__(self): + self._env_subs = None + self.name = RosWs.get_name() + + @staticmethod + def is_workspace_volume(workspace): + if os.path.isdir(os.path.expanduser(workspace)): + return True + else: + return False + + def get_docker_args(self, cli_args): + """ + @param cli_args: {'volume': [[%arg%]]} + - 'volume' is fixed. + - %arg% can be: + - %path_host%: a path on the host. Same path will be populated in + the container. + - %path_host%:%path_cont% + - %path_host%:%path_cont%:%option% + """ + workspace = cli_args[self.name] + if RosWs.is_workspace_volume(workspace): + args = Volume.get_volume_args([[workspace + ":" + os.path.join(RosWs.get_home_dir(cli_args), self.name, 'src')]]) + return ' '.join(args) + else: + return '' + + def precondition_environment(self, cli_args): + pass + + def validate_environment(self, cli_args): + pass + + def get_preamble(self, cli_args): + return "" + + def get_files(self, cli_args): + def get_files_from_path(path, only_ros_pacakges=False, is_ros_package=False): + if os.path.isdir(path): + if ( + not os.path.basename(path) == ".git" + ): # ignoring the .git directory allows the docker build context to cache the build context if the directories haven't been modified + if not is_ros_package: + is_ros_package = os.path.exists(os.path.join(path, 'package.xml')) + for basename in os.listdir(path): + yield from get_files_from_path(os.path.join(path, basename), only_ros_pacakges=only_ros_pacakges, is_ros_package=copy.copy(is_ros_package)) + else: + if not only_ros_pacakges: + yield path + if only_ros_pacakges and is_ros_package: + yield path + + def generate_ws_files(dir, only_ros_pacakges=False): + ws_files = {} + for filepath in get_files_from_path(os.path.expanduser(dir), only_ros_pacakges=only_ros_pacakges): + if os.path.islink(filepath): + # todo handle symlinks + print(f"Warning: Could not copy symlink {filepath} -> {os.readlink(filepath)}") + continue + try: + with open(filepath, "r") as f: + ws_files[filepath.replace(os.path.expanduser(dir), "ros_ws_src" + os.path.sep)] = f.read() + except UnicodeDecodeError: + # read the file as binary instead + with open(filepath, "rb") as f: + ws_files[filepath.replace(os.path.expanduser(dir), "ros_ws_src" + os.path.sep)] = f.read() + return ws_files + + workspace = cli_args[self.name] + if self.is_workspace_volume(workspace): + return generate_ws_files(workspace, only_ros_pacakges=True) + else: + # todo if rocker/docker supports ssh key passing in the build in the future, it would be better to use that inside a dockerfile + # this is a workaround to check out the repos locally and copy them include them in the build context + + # todo support workspace file when docker-py supports build kit for ssh agent forwarding + raise ValueError("Workspace file not currently supported") + + with tempfile.TemporaryDirectory() as td: + workspace_file = cli_args[self.name] + with open(workspace_file, "r") as f: + repos = yaml.safe_load(f) + for repo in repos: + vcs_type = list(repo.keys())[0] # git, hg, svn, bzr + vc = VcsClient( + vcs_type, os.path.join(td, repo[vcs_type].get("local-name", "")) + ) + vc.checkout( + repo[vcs_type]["uri"], + version=repo[vcs_type].get("version", ""), + shallow=True, + ) + + return generate_ws_files(td) + + @staticmethod + def get_rosdeps(workspace): + if RosWs.is_workspace_volume(workspace): + pass + else: + # todo support workspace file when docker-py supports build kit for ssh agent forwarding + raise ValueError("Workspace file not currently supported") + with open(workspace, "r") as f: + repos = yaml.safe_load(f) + + # Get list of package.xml files + package_xmls = [] + for root, dirs, files in os.walk(os.path.expanduser(workspace)): + if 'package.xml' in files: + package_xmls.append(os.path.join(root, 'package.xml')) + + # Parse package.xml files to get dependencies + deps = set() + src_packages = set() + for package_xml in package_xmls: + try: + tree = ET.parse(package_xml) + root = tree.getroot() + + src_packages.add(root.find('name').text.strip()) + + # Get all depend, build_depend, run_depend, etc tags + depend_tags = ['depend', 'build_depend', 'run_depend', 'exec_depend', 'test_depend'] + for tag in depend_tags: + for dep in root.findall(tag): + if dep.text: + dep_name = dep.text.strip() + deps.add(dep_name) + except ET.ParseError: + print(f"Warning: Could not parse {package_xml}") + continue + + # skip source packages from dependencies + return sorted(deps - src_packages) + + @staticmethod + def get_home_dir(cli_args): + if cli_args["user"]: + return os.path.join(os.path.sep, "home", get_user_name()) + else: + return os.path.join(os.path.sep, "root") + + def get_snippet(self, cli_args): + args = {} + args["home_dir"] = RosWs.get_home_dir(cli_args) + args["rosdeps"] = RosWs.get_rosdeps(cli_args[self.name]) + args["install_deps"] = cli_args["ros_ws_install_deps"] + + snippet = pkgutil.get_data( + "rocker", + "templates/{}_snippet.Dockerfile.em".format(self.name), + ).decode("utf-8") + return em.expand(snippet, args) + + def get_user_snippet(self, cli_args): + args = {} + args["home_dir"] = RosWs.get_home_dir(cli_args) + args["rosdeps"] = RosWs.get_rosdeps(cli_args[self.name]) + args["build_source"] = cli_args["ros_ws_build_source"] + args["install_deps"] = cli_args["ros_ws_install_deps"] + args["ros_master_uri"] = cli_args["ros_ws_ros_master_uri"] + args["build_tool_args"] = cli_args["ros_ws_build_tool_args"] + + snippet = pkgutil.get_data( + "rocker", + "templates/{}_user_snippet.Dockerfile.em".format(self.name), + ).decode("utf-8") + + print(em.expand(snippet, args)) + return em.expand(snippet, args) + + @staticmethod + def register_arguments(parser, defaults={}): + parser.add_argument( + "--ros-ws", + help="ROS workspace file. The workspace file is a yaml file that describes the ros workspace to be built in the container. It is expected that the desired $ROS_DISTRO is installed in the container and the environment variable is set (such is the case for the osrf/ros:-desktop images)", + ) + + parser.add_argument( + "--ros-ws-build-tool-args", nargs='+', default=[], help="Custom build tool args for catkin_tools (e.g. '--cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo' '--install')" + ) + + parser.add_argument( + "--ros-ws-install-deps", + action="store_true", + default=True, + help="Install ROS dependencies based on package dependencies in the workspace", + ) + + parser.add_argument( + "--ros-ws-build-source", + action="store_true", + default=True, + help="Build the source of the ROS workspace", + ) + + parser.add_argument( + "--ros-ws-ros-master-uri", + help="Specifies a ROS Master URI to set in the bashrc", + ) diff --git a/src/rocker/templates/ros_ws_snippet.Dockerfile.em b/src/rocker/templates/ros_ws_snippet.Dockerfile.em new file mode 100644 index 0000000..57a2f0d --- /dev/null +++ b/src/rocker/templates/ros_ws_snippet.Dockerfile.em @@ -0,0 +1,13 @@ +# Development-related required and optional utils +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get update \ + && apt-get install -y python3-vcstool python3-wstool python3-catkin-tools python3-pip git openssh-client \ + # Clean + && apt-get clean + +@[if install_deps] +# Workspace apt dependencies +RUN export DEBIAN_FRONTEND=noninteractive; \ + export APT_DEPS="`rosdep resolve @[for rosdep in rosdeps]@rosdep @[end for]| grep '^#apt' -A1 --no-group-separator | grep -v '^#'`" \ + && if [ ! -z "$APT_DEPS" ]; then apt-get update && apt-get install -y $APT_DEPS && apt-get clean; fi +@[end if] \ No newline at end of file diff --git a/src/rocker/templates/ros_ws_user_snippet.Dockerfile.em b/src/rocker/templates/ros_ws_user_snippet.Dockerfile.em new file mode 100644 index 0000000..d3e3ca5 --- /dev/null +++ b/src/rocker/templates/ros_ws_user_snippet.Dockerfile.em @@ -0,0 +1,62 @@ +# This snippet exists because "rosdep install ..." is very slow. It +# iterates through every package in a workspace and installs their +# dependencies one at a time; this can instead be used to aggregate +# all of the deps for an entire workspace and install them all at +# once, which is much faster. + +@[if install_deps] +# pip deps +RUN rosdep update \ + && export PIP_DEPS="`rosdep resolve @[for rosdep in rosdeps]@rosdep @[end for]| grep '^#pip' -A1 --no-group-separator | grep -v '^#'`" \ + && if [ ! -z "$PIP_DEPS" ]; then pip install $PIP_DEPS; fi +@[end if] + +RUN echo "\n# Source ROS environment" >> @home_dir/.bashrc +RUN echo ". /opt/ros/$ROS_DISTRO/setup.bash" >> @home_dir/.bashrc + +RUN mkdir -p @home_dir/ros_ws/src +WORKDIR @home_dir/ros_ws + +WORKDIR @home_dir/ros_ws/src +COPY ros_ws_src . + +WORKDIR @home_dir/ros_ws +RUN catkin init +RUN catkin config --extend /opt/ros/$ROS_DISTRO +@[for build_tool_arg in build_tool_args] +RUN catkin config @build_tool_arg +@[end for] +@[if install_deps and build_source] +# If the build, devel, install, or log spaces are located in a root directory +# they'll need to be created and re-assigned ownership to the user +#RUN for SPACE_TYPE in 'build' 'devel' 'install' 'log'; \ +# do \ +# export SPACE_DIR=$(awk "/^${SPACE_TYPE}_space:/{print \$2}" .catkin_tools/profiles/default/config.yaml); \ +# case $SPACE_DIR in \ +# /*) mkdir -p $SPACE_DIR && chown user $SPACE_DIR ;; \ +# esac \ +# done +RUN catkin build -cs; exit 0 # todo this returns success even if build fails + +RUN export INSTALL=$(awk '/^install:/{print $2}' .catkin_tools/profiles/default/config.yaml) \ + && if [ $INSTALL = 'true' ]; \ + then \ + export INSTALL_SPACE=$(awk '/^install_space:/{print $2}' .catkin_tools/profiles/default/config.yaml); \ + case $INSTALL_SPACE in \ + /*) echo ". $INSTALL_SPACE/setup.bash" >> @home_dir/.bashrc ;; \ + *) echo ". $(pwd)/$INSTALL_SPACE/setup.bash" >> @home_dir/.bashrc ;; \ + esac \ + else \ + export DEVEL_SPACE=$(awk '/^devel_space:/{print $2}' .catkin_tools/profiles/default/config.yaml); \ + case $DEVEL_SPACE in \ + /*) echo ". $DEVEL_SPACE/setup.bash" >> @home_dir/.bashrc ;; \ + *) echo ". $(pwd)/$DEVEL_SPACE/setup.bash" >> @home_dir/.bashrc ;; \ + esac \ + fi +@[end if] + +#RUN rm -fr @home_dir/ros_ws/src + +@[if ros_master_uri] +RUN echo 'export ROS_MASTER_URI="@ros_master_uri"' >> @home_dir/.bashrc +@[end if] diff --git a/src/rocker/volume_extension.py b/src/rocker/volume_extension.py index 7958afc..b578de0 100644 --- a/src/rocker/volume_extension.py +++ b/src/rocker/volume_extension.py @@ -28,36 +28,49 @@ class Volume(RockerExtension): def get_name(cls): return cls.name - def get_docker_args(self, cli_args): + @classmethod + def get_volume_args(cls, volume_args): """ - @param cli_args: {'volume': [[%arg%]]} - - 'volume' is fixed. - - %arg% can be: - - %path_host%: a path on the host. Same path will be populated in - the container. - - %path_host%:%path_cont% - - %path_host%:%path_cont%:%option% + @param volume_args: [[%arg%]] + %arg% can be: + - %path_host%: a path on the host. Same path will be populated in + the container. + - %path_host%:%path_cont% + - %path_host%:%path_cont%:%option% """ args = [''] # flatten cli_args['volume'] - volumes = [ x for sublist in cli_args[self.name] for x in sublist] + volumes = [ x for sublist in volume_args for x in sublist] for volume in volumes: elems = volume.split(':') host_dir = os.path.abspath(elems[0]) if len(elems) == 1: - args.append('{0} {1}:{1}'.format(self.ARG_DOCKER_VOLUME, host_dir)) + args.append('{0} {1}:{1}'.format(cls.ARG_DOCKER_VOLUME, host_dir)) elif len(elems) == 2: container_dir = elems[1] - args.append('{0} {1}:{2}'.format(self.ARG_DOCKER_VOLUME, host_dir, container_dir)) + args.append('{0} {1}:{2}'.format(cls.ARG_DOCKER_VOLUME, host_dir, container_dir)) elif len(elems) == 3: container_dir = elems[1] options = elems[2] - args.append('{0} {1}:{2}:{3}'.format(self.ARG_DOCKER_VOLUME, host_dir, container_dir, options)) + args.append('{0} {1}:{2}:{3}'.format(cls.ARG_DOCKER_VOLUME, host_dir, container_dir, options)) else: raise ArgumentTypeError( - '{} expects arguments in format HOST-DIR[:CONTAINER-DIR[:OPTIONS]]'.format(self.ARG_ROCKER_VOLUME)) + '{} expects arguments in format HOST-DIR[:CONTAINER-DIR[:OPTIONS]]'.format(cls.ARG_ROCKER_VOLUME)) + return args + + def get_docker_args(self, cli_args): + """ + @param cli_args: {'volume': [[%arg%]]} + - 'volume' is fixed. + - %arg% can be: + - %path_host%: a path on the host. Same path will be populated in + the container. + - %path_host%:%path_cont% + - %path_host%:%path_cont%:%option% + """ + args = self.get_volume_args(cli_args[self.name]) return ' '.join(args)