Update modules, dockerize building and testing and stop using self hosted machines

.github/workflows/reusable_test.yml

@@ -44,7 +44,7 @@ env:
 
 jobs:
   BuildAndTest:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux, X64, DockerMgBuild]
     steps:
       - name: Checkout repository and submodules
         uses: actions/checkout@v4
@@ -116,9 +116,6 @@ jobs:
       - name: Docker system prune
         run: |
           docker buildx prune --all -f
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
-          sudo rm -rf /opt/hostedtoolcache/go
-          sudo rm -rf /opt/hostedtoolcache
 
       - name: Rust library tests
         if: inputs.build_target == 'dev'
@@ -138,49 +135,21 @@ jobs:
           --load .
           docker run -d -p ${{ env.MEMGRAPH_PORT }}:7687 --name ${{ env.MAGE_CONTAINER }} memgraph-mage:${{ inputs.build_target }} --telemetry-enabled=False
 
-      - name: Set up C++
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential cmake
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PY_VERSION }}
-
       - name: Install Python test dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r ./python/tests/requirements.txt
 
-      - name: Build C++ modules
-        run: |
-          mkdir -p cpp/build
-          cd cpp/build
-          cmake ..
-          make -j${{ env.CORE_COUNT }}
-
-      - name: C++ Modules unit tests
-        if: inputs.build_target == 'dev'
-        run: |
-          cd cpp/build
-          ctest -j${{ env.CORE_COUNT }}
-
       - name: Python modules unit tests
-        if: inputs.build_target == 'dev' && inputs.build_scope != 'without ML'
-        env:
-          PYTHONPATH: "$PWD/python"
+        if: inputs.build_scope != 'without ML'
         run: |
           docker exec -i -u root ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/python/ && python3 -m pytest ."
 
       - name: Run End-to-end tests
         if: inputs.arch != 'arm64'
-        env:
-          PYTHONPATH: "$PWD/e2e"
         run: |
-          cd e2e
           docker exec -i -u root ${{ env.MAGE_CONTAINER }} bash -c \
-          "cd /mage/e2e/ && python3 -m pytest . -k 'not cugraph and not node_classification and not link_prediction and not text ${{ inputs.build_scope == 'without ML' && 'and not tgn' || '' }}'"
+          "cd /mage/e2e/ && python3 -m pytest . -k 'not cugraph ${{ inputs.build_scope == 'without ML' && 'and not tgn' || '' }}'"
 
       - name: Run End-to-end correctness tests
         if: inputs.arch != 'arm64'

Dockerfile.release

@@ -1,6 +1,7 @@
 ARG PY_VERSION_DEFAULT=3.9
+ARG OS_VERSION=bullseye
 
-FROM debian:bullseye as base
+FROM debian:${OS_VERSION} AS base
 
 USER root
 
@@ -10,29 +11,33 @@ ENV PY_VERSION ${PY_VERSION_DEFAULT}
 
 #essentials for production/dev
 RUN apt-get update && apt-get install -y \
-    libcurl4        `memgraph` \
-    libpython${PY_VERSION}   `memgraph` \
-    libssl-dev       `memgraph` \
-    openssl         `memgraph` \
-    build-essential `mage-memgraph` \
-    cmake           `mage-memgraph` \
-    curl            `mage-memgraph` \
-    g++             `mage-memgraph` \
-    python3         `mage-memgraph` \
-    python3-pip     `mage-memgraph` \
-    python3-setuptools     `mage-memgraph` \
-    python3-dev     `mage-memgraph` \
-    clang           `mage-memgraph` \
-    git             `mage-memgraph` \
-    unixodbc-dev        `mage-memgraph` \
+    libcurl4 \
+    libpython${PY_VERSION} \
+    libssl-dev \
+    openssl \
+    build-essential \
+    cmake \
+    curl \
+    g++ \
+    python3 \
+    python3-pip \
+    python3-setuptools \
+    python3-dev \
+    python${PY_VERSION}-venv \
+    clang \
+    git \
+    unixodbc-dev \
     gdb \
     procps \
     linux-perf \
     libc6-dbg \
     --no-install-recommends \
-    && ln -s /usr/bin/$(ls /usr/bin | grep perf) /usr/bin/perf \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
+RUN if [ "$OS_VERSION" = "bullseye" ]; then \
+  ln -s /usr/bin/$(ls /usr/bin | grep perf) /usr/bin/perf; \
+  fi
+
 COPY memgraph-${TARGETARCH}.deb .
 
 # Hack to remove modules that cause error on loading of torch modules
@@ -45,41 +50,57 @@ ENV LD_LIBRARY_PATH /usr/lib/memgraph/query_modules
 # Memgraph listens for Bolt Protocol on this port by default.
 EXPOSE 7687
 
-FROM base as dev
+FROM base AS dev
 
 ARG BUILD_TYPE=Release
 
 WORKDIR /mage
-COPY . /mage
-
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \
+    && export PATH="/root/.cargo/bin:${PATH}" \
+    && python3 -m venv /mage/venv
 
 #MAGE
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \
+COPY ./python/requirements.txt /mage/python/requirements.txt
+RUN . /mage/venv/bin/activate \
+    && python3 -m pip install -r /mage/python/requirements.txt
+
+COPY ./python/tests/requirements.txt /mage/python/tests/requirements.txt
+RUN . /mage/venv/bin/activate \
+    && python3 -m pip install -r /mage/python/tests/requirements.txt
+
+RUN . /mage/venv/bin/activate \
+    && python3 -m pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-2.3.1+cu121.html
+
+#DGL build from source
+RUN curl https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb --output cuda-keyring_1.1-1_all.deb \
+  && dpkg -i cuda-keyring_1.1-1_all.deb \
+  && apt-get update \
+  && apt-get -y install --no-install-recommends cuda-toolkit-12-6 \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* cuda-keyring_1.1-1_all.deb
+
+RUN git clone --recurse-submodules -b v2.3.0 https://github.com/dmlc/dgl.git  \
+    && . /mage/venv/bin/activate \
+    && cd dgl && cmake -B build -S . \
+    && cmake --build build -- -j4 \
+    && cd ./python \
+    && python3 setup.py install
+
+COPY . /mage
+RUN . /mage/venv/bin/activate \
     && export PATH="/root/.cargo/bin:${PATH}" \
-    && python3 -m  pip install -r /mage/python/requirements.txt \
-    && python3 -m  pip install -r /mage/python/tests/requirements.txt \
-    && python3 -m  pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html \
+    && export N_CORES="$(nproc 2>/dev/null || echo 1)" \
     && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ --cpp-build-flags CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+    && ctest --test-dir /mage/cpp/build -j ${N_CORES} \
     && chown -R memgraph: /mage/e2e
 
-#DGL build from source
-RUN git clone --recurse-submodules -b 0.9.x https://github.com/dmlc/dgl.git  \
-    && cd dgl && mkdir build && cd build && cmake .. \
-    && make -j4 && cd ../python && python3 setup.py install
-
-# Needs to be here and in prod phase
-# Remove modules with issues
-RUN rm /usr/lib/memgraph/query_modules/node_classification.py \
-    && rm /usr/lib/memgraph/query_modules/link_prediction.py \
-    && rm /usr/lib/memgraph/query_modules/text.so
 
 USER memgraph
 ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
 CMD [""]
 
 
 
-FROM base as prod
+FROM base AS prod
 
 USER root
 ENTRYPOINT []

README.md

@@ -238,7 +238,7 @@ curl https://sh.rustup.rs -sSf | sh -s -- -y \
 && export PATH="/root/.cargo/bin:${PATH}" \
 && python3 -m  pip install -r /mage/python/requirements.txt \
 && python3 -m  pip install -r /mage/python/tests/requirements.txt \
-&& python3 -m  pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html \
+&& python3 -m  pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-2.4.0+cu124.html \
 ```
 
 

cpp/community_detection_module/CMakeLists.txt

@@ -10,6 +10,7 @@ add_subdirectory(grappolo)
 
 add_query_module(community_detection 1 "${community_detection_module_src}")
 target_include_directories(community_detection PRIVATE ${GRAPPOLO_HEADERS} ${CMAKE_CURRENT_SOURCE_DIR})
+find_package(OpenMP REQUIRED)
 target_link_libraries(community_detection PRIVATE mg_utility basic_cd full_syn_opt coloring util OpenMP::OpenMP_CXX)
 
 ################# LabelRankT

python/requirements.txt

@@ -1,17 +1,17 @@
-gekko==0.2.8
-networkx==2.6.2
-python-Levenshtein==0.12.1
+defusedxml==0.7.1
 elasticsearch==8.4.3
+gekko==0.2.8
 gensim==4.3.3
-torch==1.12.0
-six==1.16.0
-torchmetrics==0.9.3
-igraph==0.10.2
-scikit-learn==0.24.2
 gqlalchemy==1.4.1
+igraph==0.10.2
 mysql-connector-python==8.0.32
-oracledb==1.2.2
-pyodbc==4.0.35
+networkx==2.6.2
+oracledb==2.4.1
 psycopg2-binary==2.9.9
-defusedxml==0.7.1
-scipy==1.12.0
+pyodbc==5.1.0
+python-Levenshtein==0.12.1
+scikit-learn==1.5.2
+scipy==1.13.0
+six==1.16.0
+torch==2.3.1
+torchmetrics==1.4.2

python/requirements_no_ml.txt

@@ -1,15 +1,15 @@
-gekko==0.2.8
-networkx==2.6.2
-python-Levenshtein==0.12.1
+defusedxml==0.7.1
 elasticsearch==8.4.3
+gekko==0.2.8
 gensim==4.0.0
-six==1.16.0
-igraph==0.10.2
-scikit-learn==0.24.2
 gqlalchemy==1.4.1
+igraph==0.10.2
 mysql-connector-python==8.0.32
-oracledb==1.2.2
-pyodbc==4.0.35
+networkx==2.6.2
+oracledb==2.4.1
 psycopg2-binary==2.9.9
-defusedxml==0.7.1
-scipy==1.12.0
+pyodbc==5.1.0
+python-Levenshtein==0.12.1
+scikit-learn==1.5.2
+scipy==1.13.0
+six==1.16.0

python/tests/requirements.txt

@@ -1,5 +1,5 @@
 pytest==7.0
-PyYAML==5.4.1
+PyYAML==6.0.2
 black==22.3.0
 flake8==3.9.2
 pymgclient==1.3.1
@@ -9,4 +9,4 @@ pytest-cov==2.12.1
 pytest-benchmark==3.4.1
 pytest-flake8==1.0.7
 pytest-black==0.3.12
-gqlalchemy==1.4.1
+gqlalchemy==1.4.1