diff --git a/tests/forwarding/BUILD.bazel b/tests/forwarding/BUILD.bazel index 28bf4cdf..ba4e6343 100644 --- a/tests/forwarding/BUILD.bazel +++ b/tests/forwarding/BUILD.bazel @@ -82,7 +82,6 @@ cc_library( testonly = True, srcs = ["smoke_test.cc"], hdrs = ["smoke_test.h"], - visibility = ["//visibility:public"], deps = [ ":p4_blackbox_fixture", ":test_data", diff --git a/tests/forwarding/p4_blackbox_fixture.h b/tests/forwarding/p4_blackbox_fixture.h index a0d63e25..57bbcc82 100644 --- a/tests/forwarding/p4_blackbox_fixture.h +++ b/tests/forwarding/p4_blackbox_fixture.h @@ -68,7 +68,7 @@ class P4BlackboxFixture : public thinkit::MirrorTestbedFixture { } void TearDown() override { - if (SutP4RuntimeSession() != nullptr) { + if (SutP4RuntimeSession() != nullptr && clear_table_entries_on_teardown_) { // Clear all table entries to leave the switch in a clean state. EXPECT_OK(pdpi::ClearTableEntries(SutP4RuntimeSession())); } @@ -82,9 +82,16 @@ class P4BlackboxFixture : public thinkit::MirrorTestbedFixture { const pdpi::IrP4Info& IrP4Info() const { return ir_p4info_; } + protected: + void DisableClearingTableEntriesOnTearDown() { + clear_table_entries_on_teardown_ = false; + } + private: + bool clear_table_entries_on_teardown_ = true; std::unique_ptr sut_p4rt_session_; - pdpi::IrP4Info ir_p4info_ = sai::GetIrP4Info(sai::Instantiation::kMiddleblock); + pdpi::IrP4Info ir_p4info_ = + sai::GetIrP4Info(sai::Instantiation::kMiddleblock); }; } // namespace gpins diff --git a/tests/forwarding/smoke_test.cc b/tests/forwarding/smoke_test.cc index 308bcc2a..d82fc671 100644 --- a/tests/forwarding/smoke_test.cc +++ b/tests/forwarding/smoke_test.cc @@ -129,8 +129,9 @@ TEST_P(SmokeTestFixture, InsertAndReadTableEntries) { p4::v1::ReadRequest read_request; read_request.add_entities()->mutable_table_entry(); - ASSERT_OK_AND_ASSIGN(p4::v1::ReadResponse read_response, - pdpi::SetMetadataAndSendPiReadRequest(session, read_request)); + ASSERT_OK_AND_ASSIGN( + p4::v1::ReadResponse read_response, + pdpi::SetMetadataAndSendPiReadRequest(session, read_request)); for (const auto& entity : read_response.entities()) { ASSERT_OK(test_environment.AppendToTestArtifact( diff --git a/tests/gnoi/BUILD.bazel b/tests/gnoi/BUILD.bazel index 9e72a3de..4413fec0 100644 --- a/tests/gnoi/BUILD.bazel +++ b/tests/gnoi/BUILD.bazel @@ -34,6 +34,8 @@ cc_library( "//thinkit:mirror_testbed_fixture", "@com_github_gnoi//diag:diag_cc_proto", "@com_github_google_glog//:glog", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/flags:flag", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", diff --git a/tests/gnoi/bert_tests.cc b/tests/gnoi/bert_tests.cc index 7b3327e1..def55955 100644 --- a/tests/gnoi/bert_tests.cc +++ b/tests/gnoi/bert_tests.cc @@ -14,6 +14,10 @@ #include "tests/gnoi/bert_tests.h" +#include +#include "absl/container/flat_hash_set.h" +#include "absl/flags/flag.h" + #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/substitute.h" @@ -29,6 +33,9 @@ #include "lib/gnmi/gnmi_helper.h" #include "lib/validator/validator_lib.h" +ABSL_FLAG(uint32_t, idx_seed, static_cast(std::time(nullptr)), + "Seed to randomly generate interface index."); + namespace bert { using ::google::protobuf::util::MessageDifferencer; @@ -47,6 +54,11 @@ constexpr absl::Duration kPollInterval = absl::Seconds(30); // Minimum wait time after the BERT request to read the BERT result. constexpr absl::Duration kWaitTime = absl::Seconds(30); +constexpr uint8_t kMaxAllowedInterfacesToRunBert = 96; + +constexpr char kEnabledFalse[] = "{\"enabled\":false}"; +constexpr char kEnabledTrue[] = "{\"enabled\":true}"; + const std::string BuildPerPortStartBertRequest( absl::string_view interface_name) { return absl::Substitute(R"pb( @@ -76,6 +88,65 @@ const std::string BuildOpenConfigInterface(absl::string_view interface_name) { interface_name); } +absl::StatusOr GetInterfaceNameFromOcInterfacePath( + const gnoi::types::Path& interface) { + // Validate if interface is in valid format(either in OpenConfig format or + // SONiC format). + // Example: openconfig/interfaces/interface[name=Ethernet0] or + // openconfig-interfaces/interfaces/interface[name=Ethernet0]. + std::stringstream error_stream; + if ((interface.origin() != "openconfig") && + (interface.origin() != "openconfig-interfaces")) { + error_stream << "Invalid interface origin. Expected: openconfig or " + "openconfig-interfaces but received: " + << interface.origin() << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + auto elems = interface.elem(); + if (elems.size() != 2) { + error_stream << "Invalid element path count. Expected: 2 but received: " + << elems.size() << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + if ((elems[0].name() != "interfaces") || (elems[0].key_size() > 0)) { + error_stream << "First interface path element is malformed. Expected " + "element name: interfaces but received: " + << elems[0].name() + << " and expected 0 key but received: " << elems[0].key_size() + << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + if ((elems[1].name() != "interface")) { + error_stream << "Second interface path element is malformed. Expected " + "element name: interface but received: " + << elems[1].name() << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + auto it = elems[1].key().find("name"); + if (it != elems[1].key().end()) { + return it->second; + } + return absl::InvalidArgumentError( + absl::StrCat("Interface name is missing in: ", interface.DebugString())); +} + +void SetAdminStateOnInterfaceList(gnmi::gNMI::Stub& gnmi_stub, + std::vector& interfaces, + absl::string_view value) { + for (const std::string& interface : interfaces) { + const std::string if_enabled_config_path = absl::StrCat( + "interfaces/interface[name=", interface, "]/config/enabled"); + ASSERT_OK(SetGnmiConfigPath(&gnmi_stub, if_enabled_config_path, + pins_test::GnmiSetType::kUpdate, value)); + } +} + +bool IsInterfaceInList(absl::string_view interface, + std::vector& interfaces) { + return std::find(interfaces.begin(), interfaces.end(), interface) != + interfaces.end(); +} + void VerifyBertResultForSuccess( const gnoi::diag::GetBERTResultResponse::PerPortResponse& bert_result, absl::string_view op_id, const gnoi::types::Path& interface, @@ -104,6 +175,143 @@ void VerifyBertResultForSuccess( EXPECT_EQ(bert_result.total_errors(), total_errors); } +// Helps in getting the BERT result on a set of ports and if BERT is running on +// the port, forces admin status down by disabling it. It also modifies the +// list of ports in request by removing the port that was running BERT. +void CheckRunningBertAndForceAdminDownHelper( + gnoi::diag::Diag::Stub& gnoi_diag_stub, gnmi::gNMI::Stub& gnmi_stub, + gnoi::diag::GetBERTResultRequest* request) { + gnoi::diag::GetBERTResultResponse response; + grpc::ClientContext context; + ASSERT_OK(gnoi_diag_stub.GetBERTResult(&context, *request, &response)); + + ASSERT_THAT(response.per_port_responses(), + SizeIs(request->per_port_requests_size())); + request->clear_per_port_requests(); + for (const auto& result : response.per_port_responses()) { + // Check if BERT is running. + if ((result.status() == gnoi::diag::BERT_STATUS_OK) && + (result.peer_lock_established())) { + ASSERT_OK_AND_ASSIGN( + const std::string interface, + GetInterfaceNameFromOcInterfacePath(result.interface())); + // Disable the admin status. + const std::string if_enabled_config_path = absl::StrCat( + "interfaces/interface[name=", interface, "]/config/enabled"); + ASSERT_OK(SetGnmiConfigPath(&gnmi_stub, if_enabled_config_path, + pins_test::GnmiSetType::kUpdate, + kEnabledFalse)); + } else { + // Get result for interfaces again that didn't start BERT in last poll. + *(request->add_per_port_requests()->mutable_interface()) = + result.interface(); + } + } +} + +// Checks if BERT is running on the ports where we are supposed to force admin +// status DOWN. If BERT is running, force admin status to DOWN on port. +void CheckRunningBertAndForceAdminDown( + gnoi::diag::Diag::Stub& sut_gnoi_diag_stub, + gnoi::diag::Diag::Stub& control_switch_gnoi_diag_stub, + gnmi::gNMI::Stub& sut_gnmi_stub, gnmi::gNMI::Stub& control_switch_gnmi_stub, + absl::string_view op_id, std::vector& sut_interfaces, + std::vector& control_switch_interfaces) { + gnoi::diag::GetBERTResultRequest sut_request; + sut_request.set_bert_operation_id(std::string(op_id)); + for (const std::string& interface : sut_interfaces) { + *(sut_request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface(interface)); + } + + gnoi::diag::GetBERTResultRequest control_switch_request; + control_switch_request.set_bert_operation_id(std::string(op_id)); + for (const std::string& interface : control_switch_interfaces) { + *(control_switch_request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface(interface)); + } + + int max_poll_count = + 1 + (absl::ToInt64Seconds(kSyncDuration - absl::Seconds(1)) / + absl::ToInt64Seconds(kPollInterval)); + + while (max_poll_count > 0) { + absl::SleepFor(kPollInterval); + if (sut_request.per_port_requests_size() > 0) { + // Check BERT status on SUT and force admin status down. + ASSERT_NO_FATAL_FAILURE(CheckRunningBertAndForceAdminDownHelper( + sut_gnoi_diag_stub, sut_gnmi_stub, &sut_request)); + } + + if (control_switch_request.per_port_requests_size() > 0) { + // Check BERT status on control switch and force admin status down. + ASSERT_NO_FATAL_FAILURE(CheckRunningBertAndForceAdminDownHelper( + control_switch_gnoi_diag_stub, control_switch_gnmi_stub, + &control_switch_request)); + } + if (sut_request.per_port_requests().empty() && + control_switch_request.per_port_requests().empty()) { + break; + } + --max_poll_count; + } + + EXPECT_THAT(sut_request.per_port_requests(), testing::IsEmpty()); + EXPECT_THAT(control_switch_request.per_port_requests(), testing::IsEmpty()); +} + +// Gets the BERT result on all the ports that are running BERT. Verifies BERT +// failure result on ports where admin status was forced to DOWN. Other ports +// are expected to have successful BERT results. +void GetAndVerifyBertResultsWithAdminDownInterfaces( + gnoi::diag::Diag::Stub& gnoi_diag_stub, + gnoi::diag::StartBERTRequest& bert_request, + std::vector& sut_admin_down_interfaces, + std::vector& control_switch_admin_down_interfaces) { + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id(bert_request.bert_operation_id()); + for (unsigned idx = 0; idx < bert_request.per_port_requests_size(); ++idx) { + *(result_request.add_per_port_requests()->mutable_interface()) = + bert_request.per_port_requests(idx).interface(); + } + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext result_context; + ASSERT_OK(gnoi_diag_stub.GetBERTResult(&result_context, result_request, + &result_response)); + ASSERT_THAT(result_response.per_port_responses(), + SizeIs(bert_request.per_port_requests_size())); + for (unsigned idx = 0; idx < result_response.per_port_responses_size(); + ++idx) { + // Extract interface name from OC interface path. + ASSERT_OK_AND_ASSIGN( + const std::string interface_name, + GetInterfaceNameFromOcInterfacePath( + result_response.per_port_responses(idx).interface())); + // Check if interface is part of list where admin state was disabled. + if (IsInterfaceInList(interface_name, sut_admin_down_interfaces) || + IsInterfaceInList(interface_name, + control_switch_admin_down_interfaces)) { + // Verify BERT failure. + EXPECT_EQ(result_response.per_port_responses(idx).status(), + gnoi::diag::BERT_STATUS_PEER_LOCK_LOST); + EXPECT_TRUE( + result_response.per_port_responses(idx).peer_lock_established()); + EXPECT_TRUE(result_response.per_port_responses(idx).peer_lock_lost()); + continue; + } + // If it is normal BERT running port, verify normal SUCCESS result. + VerifyBertResultForSuccess( + result_response.per_port_responses(idx), + bert_request.bert_operation_id(), + bert_request.per_port_requests(idx).interface(), + bert_request.per_port_requests(idx).prbs_polynomial()); + } +} + + + // Test StartBERT with invalid request parameters. TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { thinkit::Switch& sut = GetMirrorTestbed().Sut(); @@ -576,4 +784,129 @@ TEST_P(BertTest, StartBertSucceeds) { // ASSERT_OK(pins_test::PortsUp(control_switch)); } +// Runs the BERT test on current maximum allowed number of interfaces. During +// the BERT run: +// 1) Disable admin state of few ports on SUT, +// 2) Disable admin state of few ports on control switch, +// This helps us verify a mix of operation during BERT - unexpected software or +// hardware errors. +TEST_P(BertTest, RunBertOnMaximumAllowedPorts) { + thinkit::Switch& sut = GetMirrorTestbed().Sut(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr sut_gnmi_stub, + sut.CreateGnmiStub()); + ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sut_gnoi_diag_stub, + sut.CreateGnoiDiagStub()); + + thinkit::Switch& control_switch = GetMirrorTestbed().ControlSwitch(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr control_switch_gnmi_stub, + control_switch.CreateGnmiStub()); + ASSERT_OK(pins_test::PortsUp(control_switch)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr control_switch_gnoi_diag_stub, + control_switch.CreateGnoiDiagStub()); + + // Get all the interfaces that are operational status "UP". + ASSERT_OK_AND_ASSIGN(std::vector interfaces, + pins_test::GetUpInterfacesOverGnmi(*sut_gnmi_stub)); + // For this test, we need at least 5 UP interfaces. + ASSERT_GE(interfaces.size(), 5); + // Resize the interface list if UP ports are more than max number of allowed + // ports. + if (interfaces.size() > kMaxAllowedInterfacesToRunBert) { + interfaces.resize(kMaxAllowedInterfacesToRunBert); + } + + // Select 'N' ports on control switch and 'N' ports on SUT for admin down. + int num_interfaces_to_disable = 1 + (interfaces.size() / 16); + // Seed the std::rand(). + LOG(INFO) << "Seeding pseudo-random number generator with seed: " + << absl::GetFlag(FLAGS_idx_seed); + // Select SUT interfaces in the range [0..interfaces/2). + std::vector sut_interfaces_for_admin_down; + std::sample(interfaces.begin(), interfaces.begin() + interfaces.size() / 2, + std::back_inserter(sut_interfaces_for_admin_down), + num_interfaces_to_disable, + std::mt19937(absl::GetFlag(FLAGS_idx_seed))); + // Select control switch interfaces in the range [interfaces/2..interfaces]. + std::vector control_switch_interfaces_for_admin_down; + std::sample(interfaces.begin() + interfaces.size() / 2, interfaces.end(), + std::back_inserter(control_switch_interfaces_for_admin_down), + num_interfaces_to_disable, + std::mt19937(absl::GetFlag(FLAGS_idx_seed))); + + LOG(INFO) << "Starting BERT on " << interfaces.size() << " interfaces."; + + gnoi::diag::StartBERTRequest bert_request; + // Create the BERT request. + bert_request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + for (const std::string& interface : interfaces) { + *(bert_request.add_per_port_requests()) = + gutil::ParseProtoOrDie( + BuildPerPortStartBertRequest(interface)); + } + + // Request StartBert on the SUT switch. + { + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + EXPECT_OK( + sut_gnoi_diag_stub->StartBERT(&context, bert_request, &bert_response)); + } + + // Request StartBert on the control switch. + { + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + EXPECT_OK(control_switch_gnoi_diag_stub->StartBERT(&context, bert_request, + &bert_response)); + } + + absl::Time start_time = absl::Now(); + // Give some time to BERT to move in SYNC state. + absl::SleepFor(absl::Seconds(90)); + + absl::Time end_time = absl::Now(); + + // Poll for remaining BERT duration. + int max_poll_count = + 1 + (absl::ToInt64Seconds(kDelayDuration + kWaitTime + kSyncDuration + + kTestDuration - (end_time - start_time) - + absl::Seconds(1)) / + ToInt64Seconds(kPollInterval)); + std::vector interfaces_not_up = interfaces; + for (int count = 0; count < max_poll_count; ++count) { + absl::SleepFor(kPollInterval); + // If port is "UP" and no longer in "TESTING" oper state on both sides of + // link, BERT has completed on the link and full BERT result is ready. + for (auto it = interfaces_not_up.begin(); it != interfaces_not_up.end();) { + ASSERT_OK_AND_ASSIGN( + pins_test::OperStatus oper_status, + pins_test::GetInterfaceOperStatusOverGnmi(*sut_gnmi_stub, *it)); + if (oper_status == pins_test::OperStatus::kUp) { + ASSERT_OK_AND_ASSIGN(oper_status, + pins_test::GetInterfaceOperStatusOverGnmi( + *control_switch_gnmi_stub, *it)); + if (oper_status == pins_test::OperStatus::kUp) { + it = interfaces_not_up.erase(it); + continue; + } + } + ++it; + } + if (interfaces_not_up.empty()) break; + } + + EXPECT_THAT(interfaces_not_up, testing::IsEmpty()); + + // Wait for some time before checking the port status. + absl::SleepFor(absl::Seconds(10)); + + ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK(pins_test::PortsUp(control_switch)); +} + } // namespace bert