diff --git a/.gitmodules b/.gitmodules index caa2d250c..e6d02d44d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,4 +12,4 @@ url = https://github.com/MIC-DKFZ/nnUNet.git [submodule "tools/submission/power-dev"] path = tools/submission/power-dev - url = ../power-dev + url = https://github.com/mlcommons/power-dev.git diff --git a/loadgen/bindings/c_api.cc b/loadgen/bindings/c_api.cc index cb653c3a7..ca03a51a0 100644 --- a/loadgen/bindings/c_api.cc +++ b/loadgen/bindings/c_api.cc @@ -30,11 +30,15 @@ class SystemUnderTestTrampoline : public SystemUnderTest { SystemUnderTestTrampoline( ClientData client_data, std::string name, IssueQueryCallback issue_cb, FlushQueriesCallback flush_queries_cb, + UserConstraintsMetCallback user_constraints_met_cb, + ReportTargetQPSCallback report_target_qps_cb, ReportLatencyResultsCallback report_latency_results_cb) : client_data_(client_data), name_(std::move(name)), issue_cb_(issue_cb), flush_queries_cb_(flush_queries_cb), + user_constraints_met_cb_(user_constraints_met_cb), + report_target_qps_cb_(report_target_qps_cb), report_latency_results_cb_(report_latency_results_cb) {} ~SystemUnderTestTrampoline() override = default; @@ -46,6 +50,10 @@ class SystemUnderTestTrampoline : public SystemUnderTest { void FlushQueries() override { (*flush_queries_cb_)(); } + bool UserConstraintsMet() override { (*user_constraints_met_cb_)(); } + + void ReportTargetQPS(const double target_qps) override { (*report_target_qps_cb_)(target_qps); } + void ReportLatencyResults( const std::vector& latencies_ns) override { (*report_latency_results_cb_)(client_data_, latencies_ns.data(), @@ -57,6 +65,8 @@ class SystemUnderTestTrampoline : public SystemUnderTest { std::string name_; IssueQueryCallback issue_cb_; FlushQueriesCallback flush_queries_cb_; + UserConstraintsMetCallback user_constraints_met_cb_; + ReportTargetQPSCallback report_target_qps_cb_; ReportLatencyResultsCallback report_latency_results_cb_; }; @@ -65,10 +75,11 @@ class SystemUnderTestTrampoline : public SystemUnderTest { void* ConstructSUT(ClientData client_data, const char* name, size_t name_length, IssueQueryCallback issue_cb, FlushQueriesCallback flush_queries_cb, + UserConstraintsMetCallback user_constraints_met_cb, + ReportTargetQPSCallback report_target_qps_cb, ReportLatencyResultsCallback report_latency_results_cb) { SystemUnderTestTrampoline* sut = new SystemUnderTestTrampoline( - client_data, std::string(name, name_length), issue_cb, flush_queries_cb, - report_latency_results_cb); + client_data, std::string(name, name_length), issue_cb, flush_queries_cb, user_constraints_met_cb, report_target_qps_cb, report_latency_results_cb); return reinterpret_cast(sut); } diff --git a/loadgen/bindings/c_api.h b/loadgen/bindings/c_api.h index af6d5b38c..758f03ba1 100644 --- a/loadgen/bindings/c_api.h +++ b/loadgen/bindings/c_api.h @@ -38,6 +38,8 @@ typedef uintptr_t ClientData; typedef void (*IssueQueryCallback)(ClientData, const QuerySample*, size_t); typedef void (*FlushQueriesCallback)(); +typedef bool (*UserConstraintsMetCallback)(); +typedef void (*ReportTargetQPSCallback)(const double target_qps); typedef void (*ReportLatencyResultsCallback)(ClientData, const int64_t*, size_t); typedef void (*ResponseCallback)(ClientData, QuerySampleResponse*); @@ -55,6 +57,8 @@ void QuerySamplesCompleteResponseCb(QuerySampleResponse* responses, void* ConstructSUT(ClientData client_data, const char* name, size_t name_length, IssueQueryCallback issue_cb, FlushQueriesCallback flush_queries_cb, + UserConstraintsMetCallback user_constraints_met_cb, + ReportTargetQPSCallback report_target_qps_cb, ReportLatencyResultsCallback report_latency_results_cb); /// \brief Destroys the SUT created by ConstructSUT. void DestroySUT(void* sut); diff --git a/loadgen/bindings/python_api.cc b/loadgen/bindings/python_api.cc index 2bcf257e5..ef5774806 100644 --- a/loadgen/bindings/python_api.cc +++ b/loadgen/bindings/python_api.cc @@ -36,6 +36,8 @@ using IssueQueryCallback = std::function)>; using FastIssueQueriesCallback = std::function, std::vector)>; using FlushQueriesCallback = std::function; +using UserConstraintsMetCallback = std::function; +using ReportTargetQPSCallback = std::function; using ReportLatencyResultsCallback = std::function)>; // Forwards SystemUnderTest calls to relevant callbacks. @@ -44,10 +46,14 @@ class SystemUnderTestTrampoline : public SystemUnderTest { SystemUnderTestTrampoline( std::string name, IssueQueryCallback issue_cb, FlushQueriesCallback flush_queries_cb, + UserConstraintsMetCallback user_constraints_met_cb, + ReportTargetQPSCallback report_target_qps_cb, ReportLatencyResultsCallback report_latency_results_cb) : name_(std::move(name)), issue_cb_(issue_cb), flush_queries_cb_(flush_queries_cb), + user_constraints_met_cb_(user_constraints_met_cb), + report_target_qps_cb_(report_target_qps_cb), report_latency_results_cb_(report_latency_results_cb) {} ~SystemUnderTestTrampoline() override = default; @@ -60,6 +66,13 @@ class SystemUnderTestTrampoline : public SystemUnderTest { void FlushQueries() override { flush_queries_cb_(); } + bool UserConstraintsMet() override { return user_constraints_met_cb_(); } + + void ReportTargetQPS(const double target_qps) override { + pybind11::gil_scoped_acquire gil_acquirer; + report_target_qps_cb_(target_qps); + } + void ReportLatencyResults( const std::vector& latencies_ns) override { pybind11::gil_scoped_acquire gil_acquirer; @@ -70,6 +83,8 @@ class SystemUnderTestTrampoline : public SystemUnderTest { std::string name_; IssueQueryCallback issue_cb_; FlushQueriesCallback flush_queries_cb_; + UserConstraintsMetCallback user_constraints_met_cb_; + ReportTargetQPSCallback report_target_qps_cb_; ReportLatencyResultsCallback report_latency_results_cb_; }; @@ -78,8 +93,12 @@ class FastSystemUnderTestTrampoline : public SystemUnderTestTrampoline { FastSystemUnderTestTrampoline( std::string name, FastIssueQueriesCallback fast_issue_cb, FlushQueriesCallback flush_queries_cb, + UserConstraintsMetCallback user_constraints_met_cb, + ReportTargetQPSCallback report_target_qps_cb, ReportLatencyResultsCallback report_latency_results_cb) - : SystemUnderTestTrampoline(name, nullptr, flush_queries_cb, + : SystemUnderTestTrampoline(name, nullptr, flush_queries_cb, + user_constraints_met_cb, + report_target_qps_cb, report_latency_results_cb), fast_issue_cb_(fast_issue_cb) {} ~FastSystemUnderTestTrampoline() override = default; @@ -148,9 +167,11 @@ namespace py { uintptr_t ConstructSUT(IssueQueryCallback issue_cb, FlushQueriesCallback flush_queries_cb, + UserConstraintsMetCallback user_constraints_met_cb, + ReportTargetQPSCallback report_target_qps_cb, ReportLatencyResultsCallback report_latency_results_cb) { SystemUnderTestTrampoline* sut = new SystemUnderTestTrampoline( - "PySUT", issue_cb, flush_queries_cb, report_latency_results_cb); + "PySUT", issue_cb, flush_queries_cb, user_constraints_met_cb, report_target_qps_cb, report_latency_results_cb); return reinterpret_cast(sut); } @@ -163,9 +184,11 @@ void DestroySUT(uintptr_t sut) { uintptr_t ConstructFastSUT( FastIssueQueriesCallback fast_issue_cb, FlushQueriesCallback flush_queries_cb, + UserConstraintsMetCallback user_constraints_met_cb, + ReportTargetQPSCallback report_target_qps_cb, ReportLatencyResultsCallback report_latency_results_cb) { FastSystemUnderTestTrampoline* sut = new FastSystemUnderTestTrampoline( - "PyFastSUT", fast_issue_cb, flush_queries_cb, report_latency_results_cb); + "PyFastSUT", fast_issue_cb, flush_queries_cb, user_constraints_met_cb, report_target_qps_cb, report_latency_results_cb); return reinterpret_cast(sut); } diff --git a/loadgen/loadgen.cc b/loadgen/loadgen.cc index 950055653..f11f20419 100644 --- a/loadgen/loadgen.cc +++ b/loadgen/loadgen.cc @@ -392,6 +392,7 @@ struct PerformanceResult { double final_query_scheduled_time; // seconds from start. double final_query_issued_time; // seconds from start. double final_query_all_samples_done_time; // seconds from start. + bool user_constraints_met; // indicator of whether user defined constraints are met }; /// \brief Issues a series of pre-generated queries. @@ -403,6 +404,9 @@ PerformanceResult IssueQueries(SystemUnderTest* sut, const TestSettingsInternal& settings, const LoadableSampleSet& loaded_sample_set, SequenceGen* sequence_gen) { + // report target qps + sut->ReportTargetQPS(settings.target_qps); + // Create reponse handler. ResponseDelegateDetailed response_logger; std::uniform_real_distribution accuracy_log_offset_dist = @@ -495,6 +499,9 @@ PerformanceResult IssueQueries(SystemUnderTest* sut, // Log contention counters after every test as a sanity check. GlobalLogger().LogContentionAndAllocations(); + // Find out if user defined constrains are met + bool user_constraints_met = sut->UserConstraintsMet(); + // This properly accounts for the fact that the max completion time may not // belong to the final query. It also excludes any time spent postprocessing // in the loadgen itself after final completion, which may be significant @@ -503,10 +510,7 @@ PerformanceResult IssueQueries(SystemUnderTest* sut, GlobalLogger().GetMaxCompletionTime(); auto sut_active_duration = max_completion_time - start; LogDetail([start_for_power, sut_active_duration](AsyncDetail& detail) { - auto end_for_power = - start_for_power + - std::chrono::duration_cast( - sut_active_duration); + auto end_for_power = start_for_power + std::chrono::duration_cast(sut_active_duration); #if USE_NEW_LOGGING_FORMAT MLPERF_LOG_INTERVAL_START(detail, "power_begin", DateTimeStringForPower(start_for_power)); @@ -544,7 +548,8 @@ PerformanceResult IssueQueries(SystemUnderTest* sut, max_latency, final_query_scheduled_time, final_query_issued_time, - final_query_all_samples_done_time}; + final_query_all_samples_done_time, + user_constraints_met}; } /// \brief Wraps PerformanceResult with relevant context to change how @@ -861,7 +866,10 @@ bool PerformanceSummary::PerfConstraintsMet(std::string* recommendation) { break; case TestScenario::Server: ProcessLatencies(); - if (target_latency_percentile.sample_latency > + if (!pr.user_constraints_met) { + *recommendation = "User constraints not met. Recommend to reduce target QPS."; + perf_constraints_met = false; + } else if (target_latency_percentile.sample_latency > settings.target_latency.count()) { *recommendation = "Reduce target QPS to improve latency."; perf_constraints_met = false; diff --git a/loadgen/system_under_test.h b/loadgen/system_under_test.h index 4b98e06b1..0cda54a5d 100644 --- a/loadgen/system_under_test.h +++ b/loadgen/system_under_test.h @@ -59,6 +59,17 @@ class SystemUnderTest { /// This is especially useful in the server scenario. virtual void FlushQueries() = 0; + /// \brief Called after all requests are complete + /// in a series is made. + /// \details Clients can use this to indicate if user defined constraints + /// are met for a test. The final validity of the test will be defined + /// on both the user constraints and performance constraints + virtual bool UserConstraintsMet() = 0; + + /// \brief Reports the target qps to the SUT + /// recorded by the load generator. + virtual void ReportTargetQPS(const double target_qps) = 0; + /// \brief Reports the raw latency results to the SUT of each sample issued as /// recorded by the load generator. Units are nanoseconds. virtual void ReportLatencyResults(