Xiyon/add cb for targetqps

.gitmodules

@@ -12,4 +12,4 @@
 	url = https://github.com/MIC-DKFZ/nnUNet.git
 [submodule "tools/submission/power-dev"]
 	path = tools/submission/power-dev
-	url = ../power-dev
+	url = https://github.com/mlcommons/power-dev.git

loadgen/bindings/c_api.cc

@@ -30,11 +30,15 @@ class SystemUnderTestTrampoline : public SystemUnderTest {
   SystemUnderTestTrampoline(
       ClientData client_data, std::string name, IssueQueryCallback issue_cb,
       FlushQueriesCallback flush_queries_cb,
+      UserConstraintsMetCallback user_constraints_met_cb,
+      ReportTargetQPSCallback report_target_qps_cb,
       ReportLatencyResultsCallback report_latency_results_cb)
       : client_data_(client_data),
         name_(std::move(name)),
         issue_cb_(issue_cb),
         flush_queries_cb_(flush_queries_cb),
+        user_constraints_met_cb_(user_constraints_met_cb),
+        report_target_qps_cb_(report_target_qps_cb),
         report_latency_results_cb_(report_latency_results_cb) {}
   ~SystemUnderTestTrampoline() override = default;
 
@@ -46,6 +50,10 @@ class SystemUnderTestTrampoline : public SystemUnderTest {
 
   void FlushQueries() override { (*flush_queries_cb_)(); }
 
+  bool UserConstraintsMet() override { (*user_constraints_met_cb_)(); }
+
+  void ReportTargetQPS(const double target_qps) override { (*report_target_qps_cb_)(target_qps); }
+
   void ReportLatencyResults(
       const std::vector<QuerySampleLatency>& latencies_ns) override {
     (*report_latency_results_cb_)(client_data_, latencies_ns.data(),
@@ -57,6 +65,8 @@ class SystemUnderTestTrampoline : public SystemUnderTest {
   std::string name_;
   IssueQueryCallback issue_cb_;
   FlushQueriesCallback flush_queries_cb_;
+  UserConstraintsMetCallback user_constraints_met_cb_;
+  ReportTargetQPSCallback report_target_qps_cb_;
   ReportLatencyResultsCallback report_latency_results_cb_;
 };
 
@@ -65,10 +75,11 @@ class SystemUnderTestTrampoline : public SystemUnderTest {
 void* ConstructSUT(ClientData client_data, const char* name, size_t name_length,
                    IssueQueryCallback issue_cb,
                    FlushQueriesCallback flush_queries_cb,
+                   UserConstraintsMetCallback user_constraints_met_cb,
+                   ReportTargetQPSCallback report_target_qps_cb,
                    ReportLatencyResultsCallback report_latency_results_cb) {
   SystemUnderTestTrampoline* sut = new SystemUnderTestTrampoline(
-      client_data, std::string(name, name_length), issue_cb, flush_queries_cb,
-      report_latency_results_cb);
+      client_data, std::string(name, name_length), issue_cb, flush_queries_cb, user_constraints_met_cb, report_target_qps_cb, report_latency_results_cb);
   return reinterpret_cast<void*>(sut);
 }
 

loadgen/bindings/c_api.h

@@ -38,6 +38,8 @@ typedef uintptr_t ClientData;
 
 typedef void (*IssueQueryCallback)(ClientData, const QuerySample*, size_t);
 typedef void (*FlushQueriesCallback)();
+typedef bool (*UserConstraintsMetCallback)();
+typedef void (*ReportTargetQPSCallback)(const double target_qps);
 typedef void (*ReportLatencyResultsCallback)(ClientData, const int64_t*,
                                              size_t);
 typedef void (*ResponseCallback)(ClientData, QuerySampleResponse*);
@@ -55,6 +57,8 @@ void QuerySamplesCompleteResponseCb(QuerySampleResponse* responses,
 void* ConstructSUT(ClientData client_data, const char* name, size_t name_length,
                    IssueQueryCallback issue_cb,
                    FlushQueriesCallback flush_queries_cb,
+                   UserConstraintsMetCallback user_constraints_met_cb,
+                   ReportTargetQPSCallback report_target_qps_cb,
                    ReportLatencyResultsCallback report_latency_results_cb);
 /// \brief Destroys the SUT created by ConstructSUT.
 void DestroySUT(void* sut);

loadgen/bindings/python_api.cc

@@ -36,6 +36,8 @@ using IssueQueryCallback = std::function<void(std::vector<QuerySample>)>;
 using FastIssueQueriesCallback =
     std::function<void(std::vector<ResponseId>, std::vector<QuerySampleIndex>)>;
 using FlushQueriesCallback = std::function<void()>;
+using UserConstraintsMetCallback = std::function<bool()>;
+using ReportTargetQPSCallback = std::function<void(double)>;
 using ReportLatencyResultsCallback = std::function<void(std::vector<int64_t>)>;
 
 // Forwards SystemUnderTest calls to relevant callbacks.
@@ -44,10 +46,14 @@ class SystemUnderTestTrampoline : public SystemUnderTest {
   SystemUnderTestTrampoline(
       std::string name, IssueQueryCallback issue_cb,
       FlushQueriesCallback flush_queries_cb,
+      UserConstraintsMetCallback user_constraints_met_cb,
+      ReportTargetQPSCallback report_target_qps_cb,
       ReportLatencyResultsCallback report_latency_results_cb)
       : name_(std::move(name)),
         issue_cb_(issue_cb),
         flush_queries_cb_(flush_queries_cb),
+        user_constraints_met_cb_(user_constraints_met_cb),
+        report_target_qps_cb_(report_target_qps_cb),
         report_latency_results_cb_(report_latency_results_cb) {}
   ~SystemUnderTestTrampoline() override = default;
 
@@ -60,6 +66,13 @@ class SystemUnderTestTrampoline : public SystemUnderTest {
 
   void FlushQueries() override { flush_queries_cb_(); }
 
+  bool UserConstraintsMet() override { return user_constraints_met_cb_(); }
+
+  void ReportTargetQPS(const double target_qps) override { 
+    pybind11::gil_scoped_acquire gil_acquirer;
+    report_target_qps_cb_(target_qps);
+  }
+
   void ReportLatencyResults(
       const std::vector<QuerySampleLatency>& latencies_ns) override {
     pybind11::gil_scoped_acquire gil_acquirer;
@@ -70,6 +83,8 @@ class SystemUnderTestTrampoline : public SystemUnderTest {
   std::string name_;
   IssueQueryCallback issue_cb_;
   FlushQueriesCallback flush_queries_cb_;
+  UserConstraintsMetCallback user_constraints_met_cb_;
+  ReportTargetQPSCallback report_target_qps_cb_;
   ReportLatencyResultsCallback report_latency_results_cb_;
 };
 
@@ -78,8 +93,12 @@ class FastSystemUnderTestTrampoline : public SystemUnderTestTrampoline {
   FastSystemUnderTestTrampoline(
       std::string name, FastIssueQueriesCallback fast_issue_cb,
       FlushQueriesCallback flush_queries_cb,
+      UserConstraintsMetCallback user_constraints_met_cb,
+      ReportTargetQPSCallback report_target_qps_cb,
       ReportLatencyResultsCallback report_latency_results_cb)
-      : SystemUnderTestTrampoline(name, nullptr, flush_queries_cb,
+      : SystemUnderTestTrampoline(name, nullptr, flush_queries_cb, 
+                                  user_constraints_met_cb,
+                                  report_target_qps_cb,
                                   report_latency_results_cb),
         fast_issue_cb_(fast_issue_cb) {}
   ~FastSystemUnderTestTrampoline() override = default;
@@ -148,9 +167,11 @@ namespace py {
 
 uintptr_t ConstructSUT(IssueQueryCallback issue_cb,
                        FlushQueriesCallback flush_queries_cb,
+                       UserConstraintsMetCallback user_constraints_met_cb,
+                       ReportTargetQPSCallback report_target_qps_cb,
                        ReportLatencyResultsCallback report_latency_results_cb) {
   SystemUnderTestTrampoline* sut = new SystemUnderTestTrampoline(
-      "PySUT", issue_cb, flush_queries_cb, report_latency_results_cb);
+      "PySUT", issue_cb, flush_queries_cb, user_constraints_met_cb, report_target_qps_cb, report_latency_results_cb);
   return reinterpret_cast<uintptr_t>(sut);
 }
 
@@ -163,9 +184,11 @@ void DestroySUT(uintptr_t sut) {
 uintptr_t ConstructFastSUT(
     FastIssueQueriesCallback fast_issue_cb,
     FlushQueriesCallback flush_queries_cb,
+    UserConstraintsMetCallback user_constraints_met_cb,
+    ReportTargetQPSCallback report_target_qps_cb,
     ReportLatencyResultsCallback report_latency_results_cb) {
   FastSystemUnderTestTrampoline* sut = new FastSystemUnderTestTrampoline(
-      "PyFastSUT", fast_issue_cb, flush_queries_cb, report_latency_results_cb);
+      "PyFastSUT", fast_issue_cb, flush_queries_cb, user_constraints_met_cb, report_target_qps_cb, report_latency_results_cb);
   return reinterpret_cast<uintptr_t>(sut);
 }
 

loadgen/loadgen.cc

@@ -392,6 +392,7 @@ struct PerformanceResult {
   double final_query_scheduled_time;         // seconds from start.
   double final_query_issued_time;            // seconds from start.
   double final_query_all_samples_done_time;  // seconds from start.
+  bool user_constraints_met; // indicator of whether user defined constraints are met
 };
 
 /// \brief Issues a series of pre-generated queries.
@@ -403,6 +404,9 @@ PerformanceResult IssueQueries(SystemUnderTest* sut,
                                const TestSettingsInternal& settings,
                                const LoadableSampleSet& loaded_sample_set,
                                SequenceGen* sequence_gen) {
+  // report target qps
+  sut->ReportTargetQPS(settings.target_qps);
+
   // Create reponse handler.
   ResponseDelegateDetailed<scenario, mode> response_logger;
   std::uniform_real_distribution<double> accuracy_log_offset_dist =
@@ -495,6 +499,9 @@ PerformanceResult IssueQueries(SystemUnderTest* sut,
   // Log contention counters after every test as a sanity check.
   GlobalLogger().LogContentionAndAllocations();
 
+  // Find out if user defined constrains are met
+  bool user_constraints_met = sut->UserConstraintsMet();
+
   // This properly accounts for the fact that the max completion time may not
   // belong to the final query. It also excludes any time spent postprocessing
   // in the loadgen itself after final completion, which may be significant
@@ -503,10 +510,7 @@ PerformanceResult IssueQueries(SystemUnderTest* sut,
       GlobalLogger().GetMaxCompletionTime();
   auto sut_active_duration = max_completion_time - start;
   LogDetail([start_for_power, sut_active_duration](AsyncDetail& detail) {
-    auto end_for_power =
-        start_for_power +
-        std::chrono::duration_cast<std::chrono::system_clock::duration>(
-            sut_active_duration);
+    auto end_for_power = start_for_power + std::chrono::duration_cast<std::chrono::system_clock::duration>(sut_active_duration);
 #if USE_NEW_LOGGING_FORMAT
     MLPERF_LOG_INTERVAL_START(detail, "power_begin",
                               DateTimeStringForPower(start_for_power));
@@ -544,7 +548,8 @@ PerformanceResult IssueQueries(SystemUnderTest* sut,
                            max_latency,
                            final_query_scheduled_time,
                            final_query_issued_time,
-                           final_query_all_samples_done_time};
+                           final_query_all_samples_done_time,
+                           user_constraints_met};
 }
 
 /// \brief Wraps PerformanceResult with relevant context to change how
@@ -861,7 +866,10 @@ bool PerformanceSummary::PerfConstraintsMet(std::string* recommendation) {
       break;
     case TestScenario::Server:
       ProcessLatencies();
-      if (target_latency_percentile.sample_latency >
+      if (!pr.user_constraints_met) {
+        *recommendation = "User constraints not met. Recommend to reduce target QPS.";
+        perf_constraints_met = false;
+      } else if (target_latency_percentile.sample_latency >
           settings.target_latency.count()) {
         *recommendation = "Reduce target QPS to improve latency.";
         perf_constraints_met = false;

loadgen/system_under_test.h

@@ -59,6 +59,17 @@ class SystemUnderTest {
   /// This is especially useful in the server scenario.
   virtual void FlushQueries() = 0;
 
+  /// \brief Called after all requests are complete
+  /// in a series is made.
+  /// \details Clients can use this to indicate if user defined constraints 
+  /// are met for a test. The final validity of the test will be defined
+  /// on both the user constraints and performance constraints
+  virtual bool UserConstraintsMet() = 0;
+
+  /// \brief Reports the target qps to the SUT
+  /// recorded by the load generator.
+  virtual void ReportTargetQPS(const double target_qps) = 0;
+
   /// \brief Reports the raw latency results to the SUT of each sample issued as
   /// recorded by the load generator. Units are nanoseconds.
   virtual void ReportLatencyResults(