Merge branch 'mlcommons:master' into master

.github/workflows/build_wheels.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - master
       - loadgen-release
+      - dev
     paths:
       - loadgen/**
 
@@ -17,25 +18,28 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
 
-      # Step 3: Check if VERSION.txt file has changed in this push
+      # Check if VERSION.txt file has changed in this push
       - name: Check if VERSION.txt file has changed
         id: version_changed
         run: |
+          echo "version_changed=false" >> $GITHUB_ENV
+          echo "new_version=" >> $GITHUB_ENV  # Initialize with empty value
           if git diff --name-only HEAD~1 | grep -q "VERSION.txt"; then
             echo "VERSION.txt file has been modified"
-            echo "::set-output name=version_changed::true"
-            new_version=$(cat VERSION.txt)
+            echo "version_changed=true" >> $GITHUB_ENV
+            new_version=$(cat loadgen/VERSION.txt)
+            echo "new_version=$new_version" >> $GITHUB_ENV
           else
-            echo "VERSION file has NOT been modified"
-            echo "::set-output name=version_changed::false"
+            echo "VERSION.txt file has NOT been modified"
           fi
-          echo "::set-output name=new_version::$new_version"
 
-      # Step 4: Increment version if VERSION was not changed
+      # Step 4: Increment version if VERSION.txt was not changed
       - name: Increment version if necessary
         id: do_version_increment
-        if: steps.version_changed.outputs.version_changed == 'false'
+        if: env.version_changed == 'false'
         run: |
           cd loadgen
           # Check if VERSION file exists, else initialize it
@@ -49,17 +53,17 @@ jobs:
           new_version="$major.$minor.$patch"
           echo $new_version > VERSION.txt
           echo "New version: $new_version"
-          echo "::set-output name=new_version::$new_version"
+          echo "new_version=$new_version" >> $GITHUB_ENV
 
       # Step 5: Commit the updated version to the repository
       - name: Commit updated version
-        if: steps.version_changed.outputs.version_changed == 'false'
+        if: env.version_changed == 'false'
         run: |
           cd loadgen
           git config --global user.name "${{ github.actor }}"
           git config --global user.email "${{ github.actor }}@users.noreply.github.com"
           git add VERSION.txt
-          git commit -m "Increment version to ${{ steps.do_version_increment.outputs.new_version }}"
+          git commit -m "Increment version to $new_version"
           git push
   
   build_wheels:
@@ -80,7 +84,7 @@ jobs:
         run: python -m pip install cibuildwheel twine
 
       - name: Build wheels
-        run: python -m cibuildwheel loadgen/ --output-dir wheels
+        run: git pull && python -m cibuildwheel loadgen/ --output-dir wheels
 
        # Save wheels as artifacts
       - name: Upload built wheels

.github/workflows/format.yml

@@ -14,8 +14,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
       - name: Set up Python ${{ env.python_version }}
         uses: actions/setup-python@v3
         with:

.github/workflows/test-resnet50.yml

@@ -22,6 +22,7 @@ jobs:
       matrix:
         python-version: [ "3.9" ]
         backend: [ "onnxruntime", "tf" ]
+        loadgen-flag: [ "", "--adr.loadgen.tags=_from-pip --pip_loadgen=yes" ]
 
     steps:
     - uses: actions/checkout@v3
@@ -34,4 +35,4 @@ jobs:
         python3 -m pip install cm4mlops
     - name: Test Resnet50 and end to end submission generation
       run: |
-        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.loadgen.version=custom
+        cm run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}

.github/workflows/test-submission-checker.yml

@@ -29,7 +29,7 @@ jobs:
     - name: Install dependencies
       run: |
         python3 -m pip install cm4mlops
-        git clone https://github.com/mlcommons/inference_results_v4.0 --depth 1
+        git clone https://github.com/mlcommons/inference_results_v4.1 --depth 1
     - name: Test MLPerf inference submission checker
       run: |
-        cm run script --tags=run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --input=`pwd`/inference_results_v4.0 --src_version=v4.0 --quiet 
+        cm run script --tags=run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --input=`pwd`/inference_results_v4.1 --src_version=v4.1 --extra_args=" --skip-extra-accuracy-files-check" --quiet 

loadgen/CMakeLists.txt

@@ -1,18 +1,25 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.12)
 
 project(mlperf_loadgen)
 
 # Read the version file
 file(READ "${CMAKE_SOURCE_DIR}/VERSION.txt" VERSION_CONTENTS)
 
-# Extract the major and minor versions from the VERSION file (assuming "MAJOR.MINOR.PATCH" format)
-string(REGEX MATCH "^([0-9]+)\\.([0-9]+)" VERSION_MATCH ${VERSION_CONTENTS})
+# Extract the major, minor, and patch versions from the VERSION file (assuming "MAJOR.MINOR.PATCH" format)
+string(REGEX MATCH "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" VERSION_MATCH ${VERSION_CONTENTS})
 
-# Set the variables for the major and minor versions
+# Set the variables for the major, minor, and patch versions
 set(mlperf_loadgen_VERSION_MAJOR "${CMAKE_MATCH_1}")
 set(mlperf_loadgen_VERSION_MINOR "${CMAKE_MATCH_2}")
+set(mlperf_loadgen_VERSION_PATCH "${CMAKE_MATCH_3}")
 
-message("mlperf_loadgen v${mlperf_loadgen_VERSION_MAJOR}.${mlperf_loadgen_VERSION_MINOR}")
+# Check if the version format was parsed correctly
+if(NOT mlperf_loadgen_VERSION_MAJOR OR NOT mlperf_loadgen_VERSION_MINOR OR NOT mlperf_loadgen_VERSION_PATCH)
+    message(FATAL_ERROR "Version format in VERSION.txt is incorrect. Expected format: MAJOR.MINOR.PATCH")
+endif()
+
+# Print out the version
+message("mlperf_loadgen v${mlperf_loadgen_VERSION_MAJOR}.${mlperf_loadgen_VERSION_MINOR}.${mlperf_loadgen_VERSION_PATCH}")
 
 # Set build options. NB: CXX_STANDARD is supported since CMake 3.1.
 if (NOT MSVC)
@@ -38,6 +45,28 @@ message(STATUS "Using output path: ${LIBRARY_OUTPUT_PATH}")
 find_package(PythonInterp)
 message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}")
 
+# Specify the source and destination files
+set(CONF_FILE "mlperf.conf")
+set(HEADER_FILE "mlperf_conf.h")
+
+# Read the content of the configuration file
+file(READ ${CONF_FILE} CONF_CONTENTS)
+
+# Escape all double quotes and backslashes
+string(REPLACE "\\" "\\\\" CONF_CONTENTS "${CONF_CONTENTS}")
+string(REPLACE "\"" "\\\"" CONF_CONTENTS "${CONF_CONTENTS}")
+
+# Handle new lines
+string(REPLACE "\n" "\\n\"\n\"" CONF_CONTENTS "${CONF_CONTENTS}")
+
+# Wrap the content in a C++ string declaration
+set(FORMATTED_CONTENT "const char* mlperf_conf =\n\"${CONF_CONTENTS}\";\n")
+
+# Write the formatted content to the header file
+file(WRITE ${HEADER_FILE} "${FORMATTED_CONTENT}")
+
+message(STATUS "Output config:  ${CMAKE_BINARY_DIR}/mlperf_conf.h")
+
 # Generate source file with version info.
 execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -58,6 +87,8 @@ set(SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/results.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/version.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/version.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt
   ${CMAKE_BINARY_DIR}/version_generated.cc
 )
 
@@ -80,8 +111,3 @@ install(TARGETS mlperf_loadgen
 	DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
 	DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h")
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf
-        DESTINATION ${CMAKE_INSTALL_PREFIX}/include)
-
-# Define preprocessor macro with the path to mlperf.conf
-add_definitions(-DMLPERF_CONF_PATH=\"${CMAKE_INSTALL_PREFIX}/include/mlperf.conf\")

loadgen/MANIFEST.in

@@ -1,2 +1 @@
-include mlperf.conf
 include VERSION.txt

loadgen/VERSION.txt

@@ -1 +1 @@
-4.1.7
+4.1.23

loadgen/__init__.py

@@ -0,0 +1,4 @@
+import sys
+
+# Aliasing mlcommons_loadgen as mlperf_loadgen
+sys.modules['mlperf_loadgen'] = sys.modules[__name__]

loadgen/setup.py

@@ -29,6 +29,7 @@
 from pybind11 import get_include
 from pybind11.setup_helpers import Pybind11Extension, build_ext
 from version_generator import generate_loadgen_version_definitions
+import subprocess
 
 generated_version_source_filename = "generated/version_generated.cc"
 generate_loadgen_version_definitions(generated_version_source_filename, ".")
@@ -41,7 +42,7 @@
     "test_settings.h",
     "issue_query_controller.h",
     "early_stopping.h",
-    "query_dispatch_library.h",
+    "query_dispatch_library.h"
 ]
 
 lib_headers = [
@@ -53,6 +54,7 @@
     "results.h",
     "bindings/c_api.h",
     "version_generator.py",
+    "mlperf_conf.h"
 ]
 
 lib_sources = [
@@ -82,38 +84,53 @@
     "README.md").read_text(
         encoding="utf-8")
 
-config_file_path = Path(__file__).parent / "mlperf.conf"
-
 with open("VERSION.txt", "r") as f:
     version = f.read()
 version_split = version.split(".")
 
 if len(version_split) < 2:
-    print("Version is incomplete. Needs a format like 4.1 in VERSION file")
+    print("Version is incomplete. Needs a format like 4.1.1 in VERSION file")
+
+
+try:
+    with open("mlperf.conf", 'r') as file:
+        conf_contents = file.read()
+
+    # Escape backslashes and double quotes
+    conf_contents = conf_contents.replace('\\', '\\\\').replace('"', '\\"')
 
+    # Convert newlines
+    conf_contents = conf_contents.replace('\n', '\\n"\n"')
+
+    formatted_content = f'const char* mlperf_conf =\n"{conf_contents}";\n'
+
+    with open("mlperf_conf.h", 'w') as header_file:
+        header_file.write(formatted_content)
+
+except IOError as e:
+    raise RuntimeError(f"Failed to generate header file: {e}")
 
 mlperf_loadgen_module = Pybind11Extension(
     "mlperf_loadgen",
     define_macros=[
-        ("MAJOR_VERSION", version_split[0]),
-        ("MINOR_VERSION", version_split[1]),
-        ("MLPERF_CONF_PATH", f'"{config_file_path}"'),
+        ("MAJOR_VERSION",
+         version_split[0]),
+        ("MINOR_VERSION",
+         version_split[1])
     ],
     include_dirs=[".", get_include()],
     sources=mlperf_loadgen_sources,
     depends=mlperf_loadgen_headers,
 )
 
-setup(
-    name="mlcommons_loadgen",
-    version=version,
-    description="MLPerf Inference LoadGen python bindings",
-    url="https://mlcommons.org/",
-    cmdclass={"build_ext": build_ext},
-    ext_modules=[mlperf_loadgen_module],
-    packages=[""],
-    package_data={"": ["mlperf.conf"]},
-    include_package_data=True,
-    long_description=mlperf_long_description,
-    long_description_content_type="text/markdown",
-)
+setup(name="mlcommons_loadgen",
+      version=version,
+      description="MLPerf Inference LoadGen python bindings",
+      url="https://mlcommons.org/",
+      cmdclass={"build_ext": build_ext},
+      ext_modules=[mlperf_loadgen_module],
+      packages=['mlcommons_loadgen'],
+      package_dir={'mlcommons_loadgen': '.'},
+      include_package_data=True,
+      long_description=mlperf_long_description,
+      long_description_content_type='text/markdown')

loadgen/test_settings_internal.cc

@@ -18,6 +18,7 @@ limitations under the License.
 #include <string>
 
 #include "logging.h"
+#include "mlperf_conf.h"
 #include "utils.h"
 
 namespace mlperf {
@@ -527,7 +528,7 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model,
     if (configCount == 0) {
       // Only allow userConf as the single configFile and loadgen loads the
       // mlperfConf automatically
-      FromConfig(MLPERF_CONF_PATH, model, scenario, true);
+      FromConfig("", model, scenario, true);
     }
 
     else {
@@ -579,24 +580,33 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model,
     return true;
   };
 
-  // dirt simple config parser
-  std::ifstream fss(path);
-  std::string line;
   int line_nr = 0;
   int errors = 0;
-  if (!fss.is_open()) {
-    LogDetail([p = path](AsyncDetail &detail) {
+  // Declare the input stream before the if-else block
+  std::unique_ptr<std::istream> fss;
+  std::string line;
+
+  if (!is_mlperf_conf) {
+    // dirt simple config parser
+    fss = std::make_unique<std::ifstream>(path);
+    if (!static_cast<std::ifstream *>(fss.get())->is_open()) {
+      LogDetail([p = path](AsyncDetail &detail) {
 #if USE_NEW_LOGGING_FORMAT
-      std::stringstream ss;
-      ss << "can't open file " << p;
-      MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str());
+        std::stringstream ss;
+        ss << "can't open file " << p;
+        MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str());
 #else
-      detail.Error("can't open file ", p);
+        detail.Error("can't open file ", p);
 #endif
-    });
-    return -ENOENT;
+      });
+      return -ENOENT;
+    }
+  } else {
+    // Convert unsigned char array to std::string
+    std::string config_str(mlperf_conf);
+    fss = std::make_unique<std::istringstream>(config_str);
   }
-  while (std::getline(fss, line)) {
+  while (std::getline(*fss, line)) {
     line_nr++;
     std::istringstream iss(line);
     std::string s, k;

loadgen/version_generator.py

@@ -49,17 +49,14 @@ def func_def(name, string):
 # any modifications.
 def generate_loadgen_version_definitions_git(ofile, git_command):
     git_rev = os.popen(git_command + "rev-parse --short=10 HEAD").read()
-    git_commit_date = os.popen(git_command + 'log --format="%cI" -n 1').read()
+    git_commit_date = os.popen(
+        git_command +
+        "log --format=\"%cI\" -n 1").read()
     git_status = os.popen(git_command + "status -s -uno .").read()
     git_log = subprocess.Popen(
-        git_command + "log --pretty=oneline -n 16 --no-decorate",
-        stdout=subprocess.PIPE,
-        shell=True,
-        encoding="ascii",
-        errors="ignore",
-    ).stdout.read()
-    ofile.write(func_def("GitRevision", '"' + git_rev[0:-1] + '"'))
-    ofile.write(func_def("GitCommitDate", '"' + git_commit_date[0:-1] + '"'))
+        git_command + "log --pretty=oneline -n 16 --no-decorate", stdout=subprocess.PIPE, shell=True, encoding='ascii', errors="ignore").stdout.read()
+    ofile.write(func_def("GitRevision", "\"" + git_rev[0:-1] + "\""))
+    ofile.write(func_def("GitCommitDate", "\"" + git_commit_date[0:-1] + "\""))
     ofile.write(func_def("GitStatus", make_raw_string(git_status[0:-1])))
     ofile.write(func_def("GitLog", make_raw_string(git_log[0:-1])))
 
@@ -105,7 +102,13 @@ def generate_loadgen_version_definitions(cc_filename, loadgen_root):
     ofile.write("// DO NOT EDIT: Autogenerated by version_generator.py.\n\n")
     ofile.write("#include <string>\n\n")
     ofile.write("namespace mlperf {\n\n")
-    ofile.write(func_def("Version", '"4.1"'))
+    # Open and read the VERSION.txt file
+    with open(os.path.join(loadgen_root, "VERSION.txt"), "r") as version_file:
+        # Read and strip any extra whitespace/newlines
+        version_contents = version_file.read().strip()
+
+    # Write the version into the function definition
+    ofile.write(func_def("Version", f"\"{version_contents}\""))
 
     date_time_now_local = datetime.datetime.now().isoformat()
     date_time_now_utc = datetime.datetime.utcnow().isoformat()