Merge pull request #56 from OpenMPDK/write-optimization

Write optimization
OpenMPDK · May 7, 2024 · 005511c · 005511c
2 parents 395cafe + 5579216
commit 005511c
Show file tree

Hide file tree

Showing 216 changed files with 31,882 additions and 821 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,111 +1,27 @@
-variables:
-  BRANCH_NAME: $CI_COMMIT_BRANCH
-  SONAR_BRANCH: -Dsonar.branch.name=$CI_COMMIT_BRANCH
-  SONAR_UNIT_TEST_REPORT: df_out/reports/sonar_qube_ut_coverage_report.xml
-  COBERTURA_UNIT_TEST_REPORT: df_out/reports/cobertura.xml
-
-image:
-  name: dss-build_$BRANCH_NAME
-
-workflow:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      variables:
-        BRANCH_NAME: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
-        SONAR_BRANCH: ''
-    - if: $CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "push"
-    - if: $CI_COMMIT_BRANCH =~ /^(stable|feature)\/.*/ && $CI_PIPELINE_SOURCE == "push"
+include:
+  - local: .gitlab/defaults.yml
 
 stages:
   - build
+  - deploy
   - lint
   - test
   - scan
+  - coverage
   - archive
+  - sync
 
-build dss-sdk:
-  stage: build
-  script:
-    # Download build wrapper from local SonarQube
-    - rm -rf /build-wrapper-linux-x86
-    - wget --no-verbose --content-disposition -E -c "$SONAR_HOST_URL/static/cpp/build-wrapper-linux-x86.zip"
-    - unzip -q build-wrapper-linux-x86.zip -d /
-    # Disable ssl verify from docker build env
-    - git config --global http.sslVerify false
-    # Build client with build-wrapper
-    - /build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-output/ ./scripts/build_all.sh kdd-samsung-remote
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-  artifacts:
-    name: release build
-    expire_in: 300 seconds
-    paths:
-     - bw-output
-     - df_out
-     - host_out
-
-build target gcov and execute unit tests:
-  stage: build
-  script: ./target/build.sh --run-tests --with-coverage
-  artifacts:
-    name: gcov build
-    expire_in: 300 seconds
-    paths:
-     - df_out
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-
-target unit test coverage report:
-  stage: test
-  script:
-    - gcovr --xml $COBERTURA_UNIT_TEST_REPORT -r target/ df_out/
-    - sed -i -r 's/filename="/filename="target\//g' $COBERTURA_UNIT_TEST_REPORT
-    - sed -i -r 's/path="/path="target\//g' $SONAR_UNIT_TEST_REPORT
-  artifacts:
-    name: target unit test coverage report
-    expire_in: 300 seconds
-    paths:
-      - $SONAR_UNIT_TEST_REPORT
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: $COBERTURA_UNIT_TEST_REPORT
-  needs:
-    - build target gcov and execute unit tests
-
-sonar-scanner:
-  stage: scan
-  script:
-    # Download latest sonar-scanner from sonar-source
-    - rm -rf /sonar-scanner*
-    - wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip"
-    - unzip -q sonar-scanner-cli-*.zip -d /
-    # Scan with sonar-scanner
-    - /sonar-scanner-*-linux/bin/sonar-scanner  -Dsonar.qualitygate.wait=true -Dsonar.cfamily.build-wrapper-output=bw-output -Dsonar.coverageReportPaths=$SONAR_UNIT_TEST_REPORT
-  allow_failure: true
-  dependencies:
-    - build dss-sdk
-    - target unit test coverage report
-
-pycodestyle:
-  stage: lint
-  script: git ls-files '*.py' | xargs -n1 pycodestyle --config=./scripts/pycodestyle --show-source --show-pep8
-  needs: []
-
-shellcheck:
-  stage: lint
-  script: git ls-files '*.sh' | xargs -n1 shellcheck
-  needs: []
-
-archive dss-sdk:
-  stage: archive
-  script: /stagemergeartifacts.sh
+deploy DSS:
+  stage: deploy
   variables:
-    DSSGLOBLIST: "nkv-sdk-*.tgz nkv-target-*.tgz"
-  needs:
-    - build dss-sdk
-  artifacts:
-    name: release artifacts
-    paths:
-      - df_out/nkv-target-*.tgz
-      - host_out/nkv-sdk-bin-*.tgz
+    BRANCH_NAME: $BRANCH_NAME
+    UPSTREAM_REF: $CI_MERGE_REQUEST_REF_PATH
+  trigger:
+    include:
+      project: dfs/dss/dss-ansible
+      ref: master
+      file: .gitlab-ci.yml
+    strategy: depend
+  resource_group: inv_$CI_PROJECT_NAME.ini
+  rules:
+    - !reference [.default_rules, merge_and_push]
diff --git a/.gitlab/archive.yml b/.gitlab/archive.yml
@@ -0,0 +1,10 @@
+release tarballs:
+  stage: archive
+  script:
+    - /stagemergeartifacts.sh
+  variables:
+    DSSGLOBLIST: "nkv-sdk-*.tgz nkv-target-*.tgz"
+  needs:
+    - build dss-sdk
+  rules:
+    - !reference [.default_rules, push_only]
diff --git a/.gitlab/build.yml b/.gitlab/build.yml
@@ -0,0 +1,23 @@
+build dss-sdk:
+  stage: build
+  before_script:
+    # Download build wrapper from local SonarQube
+    - rm -rf /build-wrapper-linux-x86
+    - wget --no-verbose --content-disposition -E -c "$SONAR_HOST_URL/static/cpp/build-wrapper-linux-x86.zip"
+    - unzip -q build-wrapper-linux-x86.zip -d /
+    # Disable ssl verify from docker build env
+    - git config --global http.sslVerify false
+  script:
+    # Build client with build-wrapper
+    - /build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-output/ ./scripts/build_all.sh kdd-samsung-remote
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+  artifacts:
+    name: release build
+    expire_in: 300 seconds
+    paths:
+     - bw-output
+     - df_out
+     - host_out
+  rules:
+    - !reference [.default_rules, merge_and_push]
diff --git a/.gitlab/coverage.yml b/.gitlab/coverage.yml
@@ -0,0 +1,29 @@
+# Get the Code Coverage % from SonarQube and ingest it in GitLab
+# This stage must run after SonarQube has executed
+code coverage:
+  stage: coverage
+  image: alpine:3.19.1
+  before_script:
+    - apk add curl jq
+  script:
+    # Scrape the SonarQube Project Key from sonar-project.properties
+    - export PROJECT_KEY_LINE=$(grep -E "^sonar.projectKey=" sonar-project.properties)
+    - export PROJECT_KEY=${PROJECT_KEY_LINE//sonar.projectKey=/}
+    # Set the base URL to the SonarQube API with Project Key
+    - export SONAR_API_URL="$SONAR_HOST_URL/api/measures/component?metricKeys=coverage&component=$PROJECT_KEY"
+    # Conditionally add branch or pullRequest argument if this is an MR
+    - |
+        if [ "$CI_PIPELINE_SOURCE" == 'merge_request_event' ]; then
+            export SONAR_API_URL="$SONAR_API_URL&pullRequest=$CI_MERGE_REQUEST_IID"
+        else
+            export SONAR_API_URL="$SONAR_API_URL&branch=$CI_COMMIT_REF_NAME"
+        fi
+    # Query the API and use jq to parse the code coverage %
+    - export PERCENT=$(curl -s -u "$SONAR_API_TOKEN:" "$SONAR_API_URL" | jq '.component.measures[0].value')
+    # Print the coverage string to console so that we can scrape it with GitLab
+    - echo "SonarQube Coverage Percent ${PERCENT//\"/}"
+  coverage: '/^SonarQube Coverage Percent (.+)$/'
+  needs:
+    - sonar-scanner
+  rules:
+    - !reference [.default_rules, merge_and_push]
diff --git a/.gitlab/defaults.yml b/.gitlab/defaults.yml
@@ -0,0 +1,18 @@
+include:
+  - project: dfs/dss/dss
+    ref: master
+    file: .gitlab/defaults.yml
+  - project: dfs/dss/dss
+    ref: master
+    file: .gitlab/sync-github.yml
+  - .gitlab/archive.yml
+  - .gitlab/build.yml
+  - .gitlab/coverage.yml
+  - .gitlab/pycodestyle.yml
+  - .gitlab/shellcheck.yml
+  - .gitlab/sonar.yml
+  - .gitlab/unittest.yml
+
+sync-github:
+  variables:
+    GITHUB_REPO: dss-sdk
diff --git a/.gitlab/pycodestyle.yml b/.gitlab/pycodestyle.yml
@@ -0,0 +1,6 @@
+pycodestyle:
+  stage: lint
+  script: git ls-files '*.py' | xargs -n1 pycodestyle --config=./scripts/pycodestyle --show-source --show-pep8
+  needs: []
+  rules:
+    - !reference [.default_rules, merge_and_push]
diff --git a/.gitlab/shellcheck.yml b/.gitlab/shellcheck.yml
@@ -0,0 +1,8 @@
+include:
+  - project: dfs/dss/dss
+    ref: master
+    file: .gitlab/shellcheck.yml
+
+shellcheck:
+  variables:
+    SHELLCHECK_VER: v0.8.0
diff --git a/.gitlab/sonar.yml b/.gitlab/sonar.yml
@@ -0,0 +1,19 @@
+sonar-scanner:
+  stage: scan
+  before_script:
+    # Download latest sonar-scanner from sonar-source
+    - rm -rf /sonar-scanner*
+    - wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip"
+    - unzip -q sonar-scanner-cli-*.zip -d /
+  script:
+    # Scan with sonar-scanner
+    - /sonar-scanner-*-linux/bin/sonar-scanner  -Dsonar.qualitygate.wait=true -Dsonar.cfamily.build-wrapper-output=bw-output -Dsonar.coverageReportPaths=$SONAR_UNIT_TEST_REPORT
+  allow_failure: true
+  needs:
+    - build dss-sdk
+    - target unit test
+  variables:
+    GIT_DEPTH: 0
+    SONAR_UNIT_TEST_REPORT: df_out/reports/sonar_qube_ut_coverage_report.xml
+  rules:
+    - !reference [.default_rules, merge_and_push]
diff --git a/.gitlab/unittest.yml b/.gitlab/unittest.yml
@@ -0,0 +1,17 @@
+target unit test:
+  stage: test
+  script: ./target/build.sh --run-tests --with-coverage
+  needs: []
+  artifacts:
+    name: target unit test reports
+    expire_in: 300 seconds
+    paths:
+     - df_out/reports
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: df_out/reports/cobertura.xml
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+  rules:
+    - !reference [.default_rules, merge_and_push]
diff --git a/buildspec/sonar-scanner.yml b/buildspec/sonar-scanner.yml
@@ -16,15 +16,18 @@ phases:
       - aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/build" . --only-show-errors
       # Download unit test coverage report(s)
       - aws s3 cp --recursive "$DSSS3URI/cache/dss-sdk/$GITHUB_RUN_NUMBER/unit/df_out/reports/" df_out/reports/ --only-show-errors
-      # Correct paths of target files in coverage report
-      - sed -i -r 's/path="/path="target\//g' $SONAR_UNIT_TEST_REPORT
       # replace the old CODEBUILD_SRC_DIR with the current one in bw-output
       - sed -i -r "s|/codebuild/output/src[^/]+/src/github.com/OpenMPDK/dss-sdk|$CODEBUILD_SRC_DIR|g" bw-output/build-wrapper-dump.json
+      # Download the latest sonar-scanner
+      - rm -rf /sonar-scanner*
+      - wget --no-verbose --content-disposition -E -c "https://search.maven.org/remote_content?g=org.sonarsource.scanner.cli&a=sonar-scanner-cli&v=LATEST&c=linux&e=zip"
+      - unzip -q sonar-scanner-cli-*.zip -d /
+      - rm -f sonar-scanner-cli*.zip
   build:
     commands:
       # Run sonar-scanner and ingest coverage report(s)
       - |
-        sonar-scanner \
+        /sonar-scanner-*-linux/bin/sonar-scanner \
           -Dsonar.branch.name="$([[ "$GITHUB_REF_NAME" != *"/merge" ]] && echo "$GITHUB_REF_NAME")" \
           -Dsonar.host.url=https://sonarcloud.io \
           -Dsonar.pullrequest.github.summary_comment=true \

diff --git a/host/scripts/minio_cleanup.sh b/host/scripts/minio_cleanup.sh
@@ -1,13 +1,74 @@
 #!/usr/bin/env bash
-export LD_LIBRARY_PATH=../lib
-start_drive=$1
-end_drive=$2
-conf_file=$3
-[ $# -lt 2 ] && { echo "Usage: $0 <start_drive> <end_drive> <nkv_config.json>"; exit 1; }
-for i in $(eval echo "{$start_drive..$end_drive}");
-do 
-    ./nkv_test_cli -c ../conf/"$conf_file" -i msl-ssg-dl04 -p 1030 -w meta/.minio.sys/format.json -s /dev/nvme"${i}"n1 -o 2 -v 2097152
-    ./nkv_test_cli -c ../conf/"$conf_file" -i msl-ssg-dl04 -p 1030 -w meta/.minio.sys/kv-volumes -s /dev/nvme"${i}"n1 -o 2 -v 2097152
-    ./nkv_test_cli -c ../conf/"$conf_file" -i msl-ssg-dl04 -p 1030 -w meta/.minio.sys/config/config.json/xl.json -s /dev/nvme"${i}"n1 -o 2 -v 2097152
+# shellcheck disable=SC2059
+set -e
+
+# Set paths
+HOST_BIN_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+HOST_CONF_DIR="$(realpath "$HOST_BIN_DIR/../conf")"
+
+# Default Values
+PORT=1030
+VAL_SIZE=2097152
+OPERATION=2
+HOSTNAME=$(hostname)
+
+# Mandatory Values (no default value)
+START_DRIVE="$1"
+END_DRIVE="$2"
+CONF_FILE="$3"
+
+# Parse command line arguments
+while getopts ":s:e:c:p:v:o" opt; do
+  case $opt in
+    s) START_DRIVE="$OPTARG";;
+    e) END_DRIVE="$OPTARG";;
+    c) CONF_FILE="$OPTARG";;
+    p) PORT="$OPTARG";;
+    v) VAL_SIZE="$OPTARG";;
+    o) OPERATION="$OPTARG";;
+    \?) echo "Invalid option: -$OPTARG" >&2;;
+  esac
 done
 
+# Print help if manditory args not provided or not sudoer
+if [[ -z "$START_DRIVE" || -z "$END_DRIVE" || -z "$CONF_FILE" || "$EUID" -ne 0 ]]
+then
+  echo "Cleanup DSS MinIO metadata"
+  echo "Usage: $0"
+  echo "*** NOTE: Must run as root or sudoer ***"
+  echo ""
+  echo "Options:"
+  cols="%-35s%s\n"
+  printf "$cols" '  -s START_DRIVE'                     "Starting index of mounted NVMeOF subsystem (inclusive)"
+  printf "$cols" '  -e END_DRIVE'                       "Ending index of mounted NVMeOF subsystem (inclusive)"
+  printf "$cols" '  -c CONF_FILE'                       "Absolute path to nkv_config.json configuration file."
+  printf "$cols" ' '                                    "If only filename, file is assumed to be in $HOST_CONF_DIR"
+  printf "$cols" "  -p PORT (Default: $PORT)"           "DSS Target port"
+  printf "$cols" "  -v VAL_SIZE (Default: $VAL_SIZE)"   "KV Value size to delete"
+  printf "$cols" "  -o OPERATION (Default: $OPERATION)" "KV operation to perform"
+  exit 1
+fi
+
+# Determine path to conf file
+if [[ "$CONF_FILE" =~ ^/ ]]
+then
+    echo "Absolute path to CONF_FILE provided."
+else
+    echo "CONF_FILE provided without absolute path. Assumed to be located in $HOST_CONF_DIR"
+    CONF_FILE="$HOST_CONF_DIR/$CONF_FILE"
+fi
+
+# chdir to host bin dir - nkv_test_cli only works from its PWD
+pushd "$HOST_BIN_DIR"
+# Loop over each NVMeOF subsystem disk number
+for i in $(eval echo "{$START_DRIVE..$END_DRIVE}")
+do
+    # Loop over each key to delete
+    for KEY in 'format.json' 'kv-volumes' 'config/config.json/xl.json'
+    do
+        NKV_TEST_CLI_CMD="./nkv_test_cli -c $CONF_FILE -i $HOSTNAME -p $PORT -w meta/.minio.sys/$KEY -s /dev/nvme${i}n1 -o $OPERATION -v $VAL_SIZE"
+        echo "*** Executing command: $NKV_TEST_CLI_CMD"
+        eval "$NKV_TEST_CLI_CMD"
+    done
+done
+popd