From 3c304f959edeb050425153b3178431ecdcedaccb Mon Sep 17 00:00:00 2001 From: smolkaj Date: Mon, 28 Oct 2024 00:15:48 -0700 Subject: [PATCH 1/4] Implement the changes for port name update on test side. Add a test where StartBert fails to get the lock with its peer if StartBert is requested only on one side of the link. --- tests/forwarding/BUILD.bazel | 1 + tests/forwarding/p4_blackbox_fixture.h | 17 ++++++- tests/forwarding/smoke_test.cc | 4 +- tests/gnoi/BUILD.bazel | 1 + tests/gnoi/bert_tests.cc | 69 +++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 5 deletions(-) diff --git a/tests/forwarding/BUILD.bazel b/tests/forwarding/BUILD.bazel index 6431e3be..5b3b08c8 100644 --- a/tests/forwarding/BUILD.bazel +++ b/tests/forwarding/BUILD.bazel @@ -66,6 +66,7 @@ cc_library( hdrs = ["p4_blackbox_fixture.h"], deps = [ "//gutil:status_matchers", + "//lib/gnmi:gnmi_helper", "//p4_pdpi:p4_runtime_session", "//p4_pdpi:pd", "//sai_p4/instantiations/google:sai_p4info_cc", diff --git a/tests/forwarding/p4_blackbox_fixture.h b/tests/forwarding/p4_blackbox_fixture.h index fe3e6178..a0d63e25 100644 --- a/tests/forwarding/p4_blackbox_fixture.h +++ b/tests/forwarding/p4_blackbox_fixture.h @@ -18,6 +18,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "gutil/status_matchers.h" +#include "lib/gnmi/gnmi_helper.h" #include "p4_pdpi/p4_runtime_session.h" #include "p4_pdpi/pd.h" #include "sai_p4/instantiations/google/sai_p4info.h" @@ -35,12 +36,24 @@ class P4BlackboxFixture : public thinkit::MirrorTestbedFixture { public: void SetUp() override { MirrorTestbedFixture::SetUp(); + + thinkit::MirrorTestbed& testbed = + GetParam().mirror_testbed->GetMirrorTestbed(); + + // Get a gNMI config from the switch to use for testing. + ASSERT_OK_AND_ASSIGN(auto sut_gnmi_stub, testbed.Sut().CreateGnmiStub()); + ASSERT_OK_AND_ASSIGN(std::string sut_gnmi_config, + pins_test::GetGnmiConfig(*sut_gnmi_stub)); + // Push the gnmi configuration. + ASSERT_OK( + pins_test::PushGnmiConfig(GetMirrorTestbed().Sut(), sut_gnmi_config)); + ASSERT_OK(pins_test::PushGnmiConfig(GetMirrorTestbed().ControlSwitch(), + sut_gnmi_config)); + // Initialize the connection. ASSERT_OK_AND_ASSIGN(sut_p4rt_session_, pdpi::P4RuntimeSession::Create( GetMirrorTestbed().Sut())); - //ASSERT_OK(pdpi::SetForwardingPipelineConfig(sut_p4rt_session_.get(), - // sai::GetP4Info(sai::Instantiation::kMiddleblock))); ASSERT_OK(pdpi::SetMetadataAndSetForwardingPipelineConfig(sut_p4rt_session_.get(), p4::v1::SetForwardingPipelineConfigRequest::RECONCILE_AND_COMMIT, sai::GetP4Info(sai::Instantiation::kMiddleblock))); diff --git a/tests/forwarding/smoke_test.cc b/tests/forwarding/smoke_test.cc index 05ae2ebd..fed58129 100644 --- a/tests/forwarding/smoke_test.cc +++ b/tests/forwarding/smoke_test.cc @@ -34,7 +34,7 @@ TEST_P(SmokeTestFixture, InsertTableEntry) { router_interface_table_entry { match { router_interface_id: "router-interface-1" } action { - set_port_and_src_mac { port: "0x000" src_mac: "02:2a:10:00:00:03" } + set_port_and_src_mac { port: "1" src_mac: "02:2a:10:00:00:03" } } } )PB"); @@ -50,7 +50,7 @@ TEST_P(SmokeTestFixture, InsertTableEntryWithRandomCharacterId) { router_interface_table_entry { match { router_interface_id: "\x01\x33\x00\xff,\":'}(*{+-" } action { - set_port_and_src_mac { port: "0x000" src_mac: "02:2a:10:00:00:03" } + set_port_and_src_mac { port: "1" src_mac: "02:2a:10:00:00:03" } } } )PB"); diff --git a/tests/gnoi/BUILD.bazel b/tests/gnoi/BUILD.bazel index dbbdb955..89f015c1 100644 --- a/tests/gnoi/BUILD.bazel +++ b/tests/gnoi/BUILD.bazel @@ -29,6 +29,7 @@ cc_library( deps = [ "//gutil:status_matchers", "//gutil:testing", + "//lib/gnmi:gnmi_helper", "//thinkit:mirror_testbed_fixture", "@com_github_gnoi//diag:diag_cc_proto", "@com_github_google_glog//:glog", diff --git a/tests/gnoi/bert_tests.cc b/tests/gnoi/bert_tests.cc index 10411f67..f4a551d2 100644 --- a/tests/gnoi/bert_tests.cc +++ b/tests/gnoi/bert_tests.cc @@ -24,17 +24,20 @@ #include "gtest/gtest.h" #include "gutil/status_matchers.h" #include "gutil/testing.h" +#include "lib/gnmi/gnmi_helper.h" namespace bert { using ::testing::HasSubstr; +using ::testing::SizeIs; // Test StartBERT with invalid request parameters. TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { thinkit::Switch& sut = GetMirrorTestbed().Sut(); + ASSERT_OK_AND_ASSIGN(auto sut_gnmi_stub, sut.CreateGnmiStub()); ASSERT_OK_AND_ASSIGN(auto sut_gnoi_diag_stub, sut.CreateGnoiDiagStub()); - // TODO (b/182417612) : Select one admin state "up" port. + // TODO (b/182417612) : Select one operational state "up" port. gnoi::diag::StartBERTRequest valid_request = gutil::ParseProtoOrDie(R"PROTO( bert_operation_id: "OpId-1" @@ -128,4 +131,68 @@ TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { } } +// Test StartBERT fails if peer port is not running BERT. +TEST_P(BertTest, StartBertfailsIfPeerPortNotRunningBert) { + thinkit::Switch& sut = GetMirrorTestbed().Sut(); + ASSERT_OK_AND_ASSIGN(auto sut_gnmi_stub, sut.CreateGnmiStub()); + ASSERT_OK_AND_ASSIGN(auto sut_gnoi_diag_stub, sut.CreateGnoiDiagStub()); + // TODO : Select one operational state "up" port. + gnoi::diag::StartBERTRequest bert_request = + gutil::ParseProtoOrDie(R"PROTO( + bert_operation_id: "OpId-1" + per_port_requests { + interface { + origin: "openconfig" + elem { name: "interfaces" } + elem { + name: "interface" + key { key: "name" value: "Ethernet0" } + } + } + prbs_polynomial: PRBS_POLYNOMIAL_PRBS23 + test_duration_in_secs: 180 + } + )PROTO"); + // Set a random BERT operation id. + bert_request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + LOG(INFO) << "Sending StartBERT request: " << bert_request.ShortDebugString(); + EXPECT_OK( + sut_gnoi_diag_stub->StartBERT(&context, bert_request, &bert_response)); + // Poll for a maximum of 10 minutes to check if BERT is completed. + constexpr absl::Duration kPollInterval = absl::Seconds(30); + constexpr int kMaxPollCount = 20; + bool poll_timeout = true; + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id(bert_request.bert_operation_id()); + result_request.add_per_port_requests()->mutable_interface()->CopyFrom( + bert_request.per_port_requests(0).interface()); + for (int count = 0; count < kMaxPollCount; ++count) { + absl::SleepFor(kPollInterval); + auto statusor = + pins_test::GetInterfaceOperStatusOverGnmi(*sut_gnmi_stub, "Ethernet0"); + // If port is "UP" and no longer in "TESTING" oper state, BERT has completed + // on the port and full BERT result is ready for read. + if (statusor.ok() && (*statusor == pins_test::OperStatus::kUp)) { + poll_timeout = false; + break; + } + } + if (poll_timeout == true) { + LOG(WARNING) + << "BERT is not completed on the port in maximum allowed time."; + } + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext result_context; + EXPECT_OK(sut_gnoi_diag_stub->GetBERTResult(&result_context, result_request, + &result_response)); + LOG(INFO) << "Result: " << result_response.ShortDebugString(); + // Verify the response. + ASSERT_THAT(result_response.per_port_responses(), SizeIs(1)); + EXPECT_EQ(result_response.per_port_responses(0).status(), + gnoi::diag::BERT_STATUS_PEER_LOCK_FAILURE); +} + } // namespace bert From 43d962e82e6a60e94d294be3ee342673cbc1c0cc Mon Sep 17 00:00:00 2001 From: Srikishen Pondicherry Shanmugam Date: Mon, 28 Oct 2024 03:05:47 -0700 Subject: [PATCH 2/4] Add additional BERT tests. --- tests/forwarding/smoke_test.cc | 8 +- tests/forwarding/test_data.cc | 4 +- tests/gnoi/BUILD.bazel | 1 + tests/gnoi/bert_tests.cc | 371 ++++++++++++++++++++++++++++----- 4 files changed, 323 insertions(+), 61 deletions(-) diff --git a/tests/forwarding/smoke_test.cc b/tests/forwarding/smoke_test.cc index fed58129..cbefe6a7 100644 --- a/tests/forwarding/smoke_test.cc +++ b/tests/forwarding/smoke_test.cc @@ -30,14 +30,14 @@ namespace { TEST_P(SmokeTestFixture, InsertTableEntry) { const sai::TableEntry pd_entry = gutil::ParseProtoOrDie( - R"PB( + R"pb( router_interface_table_entry { match { router_interface_id: "router-interface-1" } action { set_port_and_src_mac { port: "1" src_mac: "02:2a:10:00:00:03" } } } - )PB"); + )pb"); ASSERT_OK_AND_ASSIGN(const p4::v1::TableEntry pi_entry, pdpi::PartialPdTableEntryToPiTableEntry(IrP4Info(), pd_entry)); @@ -46,14 +46,14 @@ TEST_P(SmokeTestFixture, InsertTableEntry) { TEST_P(SmokeTestFixture, InsertTableEntryWithRandomCharacterId) { sai::TableEntry pd_entry = gutil::ParseProtoOrDie( - R"PB( + R"pb( router_interface_table_entry { match { router_interface_id: "\x01\x33\x00\xff,\":'}(*{+-" } action { set_port_and_src_mac { port: "1" src_mac: "02:2a:10:00:00:03" } } } - )PB"); + )pb"); ASSERT_OK_AND_ASSIGN(const p4::v1::TableEntry pi_entry, pdpi::PartialPdTableEntryToPiTableEntry(IrP4Info(), pd_entry)); diff --git a/tests/forwarding/test_data.cc b/tests/forwarding/test_data.cc index be71dd2c..fc50004a 100644 --- a/tests/forwarding/test_data.cc +++ b/tests/forwarding/test_data.cc @@ -32,7 +32,7 @@ std::vector CreateUpTo255GenericTableEntries( for (int i = 0; i < num_table_entries; ++i) { sai::TableEntry& pd_entry = pd_table_entries.emplace_back(); pd_entry = gutil::ParseProtoOrDie( - R"PB( + R"pb( acl_ingress_table_entry { match { is_ipv4 { value: "0x1" } @@ -43,7 +43,7 @@ std::vector CreateUpTo255GenericTableEntries( action { trap { qos_queue: "0x1" } } priority: 2040 } - )PB"); + )pb"); pd_entry.mutable_acl_ingress_table_entry() ->mutable_match() ->mutable_dst_ip() diff --git a/tests/gnoi/BUILD.bazel b/tests/gnoi/BUILD.bazel index 89f015c1..e62ea457 100644 --- a/tests/gnoi/BUILD.bazel +++ b/tests/gnoi/BUILD.bazel @@ -30,6 +30,7 @@ cc_library( "//gutil:status_matchers", "//gutil:testing", "//lib/gnmi:gnmi_helper", + "//lib/validator:validator_lib", "//thinkit:mirror_testbed_fixture", "@com_github_gnoi//diag:diag_cc_proto", "@com_github_google_glog//:glog", diff --git a/tests/gnoi/bert_tests.cc b/tests/gnoi/bert_tests.cc index f4a551d2..a1ba8b27 100644 --- a/tests/gnoi/bert_tests.cc +++ b/tests/gnoi/bert_tests.cc @@ -15,48 +15,100 @@ #include "tests/gnoi/bert_tests.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "absl/strings/substitute.h" #include "absl/time/clock.h" #include "absl/time/time.h" #include "diag/diag.pb.h" #include "glog/logging.h" #include "google/protobuf/text_format.h" +#include "google/protobuf/util/message_differencer.h" #include "gtest/gtest.h" #include "gutil/status_matchers.h" #include "gutil/testing.h" #include "lib/gnmi/gnmi_helper.h" +#include "lib/validator/validator_lib.h" namespace bert { +using ::google::protobuf::util::MessageDifferencer; using ::testing::HasSubstr; using ::testing::SizeIs; +// BERT test duration. +constexpr absl::Duration kTestDuration = absl::Seconds(180); +// Maximum allowed duration for port to sync with its peer. +constexpr absl::Duration kSyncDuration = absl::Minutes(5); +// Maximum allowed BERT delay duration due to setup, sync and recovery +// operations. +constexpr absl::Duration kDelayDuration = absl::Minutes(10); +// Polling interval. +constexpr absl::Duration kPollInterval = absl::Seconds(30); +// Minimum wait time after the BERT request to read the BERT result. +constexpr absl::Duration kWaitTime = absl::Seconds(30); + +const std::string BuildPerPortStartBertRequest( + absl::string_view interface_name) { + return absl::Substitute(R"pb( + interface { + origin: "openconfig" + elem { name: "interfaces" } + elem { + name: "interface" + key { key: "name" value: '$0' } + } + } + prbs_polynomial: PRBS_POLYNOMIAL_PRBS23 + test_duration_in_secs: $1 + )pb", + interface_name, ToInt64Seconds(kTestDuration)); +} + +void VerifyBertResultForSuccess( + const gnoi::diag::GetBERTResultResponse::PerPortResponse& bert_result, + absl::string_view op_id, const gnoi::types::Path& interface, + gnoi::diag::PrbsPolynomial prbs_order) { + EXPECT_EQ(bert_result.status(), gnoi::diag::BERT_STATUS_OK); + EXPECT_TRUE(MessageDifferencer::Equals(bert_result.interface(), interface)); + EXPECT_EQ(bert_result.bert_operation_id(), op_id); + EXPECT_EQ(bert_result.prbs_polynomial(), prbs_order); + EXPECT_TRUE(bert_result.peer_lock_established()); + EXPECT_FALSE(bert_result.peer_lock_lost()); + // Check the timestamps to verify if time taken for BERT is between test + // duration and (test duration + 60 seconds). + EXPECT_GE(bert_result.last_bert_get_result_timestamp() - + bert_result.last_bert_start_timestamp(), + absl::ToInt64Microseconds(kTestDuration)); + EXPECT_LE(bert_result.last_bert_get_result_timestamp() - + bert_result.last_bert_start_timestamp(), + absl::ToInt64Microseconds(kTestDuration + absl::Seconds(60))); + + EXPECT_THAT(bert_result.error_count_per_minute(), + SizeIs(absl::ToInt64Minutes(kTestDuration))); + uint64_t total_errors = 0; + for (const uint32_t error_count : bert_result.error_count_per_minute()) { + total_errors += error_count; + } + EXPECT_EQ(bert_result.total_errors(), total_errors); +} + // Test StartBERT with invalid request parameters. TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { thinkit::Switch& sut = GetMirrorTestbed().Sut(); - ASSERT_OK_AND_ASSIGN(auto sut_gnmi_stub, sut.CreateGnmiStub()); - ASSERT_OK_AND_ASSIGN(auto sut_gnoi_diag_stub, sut.CreateGnoiDiagStub()); + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sut_gnoi_diag_stub, + sut.CreateGnoiDiagStub()); // TODO (b/182417612) : Select one operational state "up" port. - gnoi::diag::StartBERTRequest valid_request = - gutil::ParseProtoOrDie(R"PROTO( - bert_operation_id: "OpId-1" - per_port_requests { - interface { - origin: "openconfig" - elem { name: "interfaces" } - elem { - name: "interface" - key { key: "name" value: "Ethernet0" } - } - } - prbs_polynomial: PRBS_POLYNOMIAL_PRBS23 - test_duration_in_secs: 600 - } - )PROTO"); - // Set a unique BERT operation id using current time. + gnoi::diag::StartBERTRequest valid_request; + // Create the BERT request. valid_request.set_bert_operation_id( absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + *(valid_request.add_per_port_requests()) = + gutil::ParseProtoOrDie( + BuildPerPortStartBertRequest("Ethernet0")); gnoi::diag::StartBERTResponse response; // Case 1: BERT test duration is 0 secs. @@ -109,17 +161,16 @@ TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { gnoi::diag::StartBERTRequest invalid_interface_request = valid_request; gnoi::types::Path invalid_interface = gutil::ParseProtoOrDie( - R"PROTO( + R"pb( origin: "openconfig" elem { name: "interfaces" } elem { name: "interface" key { key: "name" value: "InvalidPort" } } - )PROTO"); - invalid_interface_request.mutable_per_port_requests(0) - ->mutable_interface() - ->CopyFrom(invalid_interface); + )pb"); + *(invalid_interface_request.mutable_per_port_requests(0) + ->mutable_interface()) = invalid_interface; response.Clear(); grpc::ClientContext context; LOG(INFO) << "Sending StartBERT request: " @@ -129,53 +180,48 @@ TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { gutil::StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("Interface is invalid"))); } + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); } // Test StartBERT fails if peer port is not running BERT. TEST_P(BertTest, StartBertfailsIfPeerPortNotRunningBert) { thinkit::Switch& sut = GetMirrorTestbed().Sut(); - ASSERT_OK_AND_ASSIGN(auto sut_gnmi_stub, sut.CreateGnmiStub()); - ASSERT_OK_AND_ASSIGN(auto sut_gnoi_diag_stub, sut.CreateGnoiDiagStub()); - // TODO : Select one operational state "up" port. - gnoi::diag::StartBERTRequest bert_request = - gutil::ParseProtoOrDie(R"PROTO( - bert_operation_id: "OpId-1" - per_port_requests { - interface { - origin: "openconfig" - elem { name: "interfaces" } - elem { - name: "interface" - key { key: "name" value: "Ethernet0" } - } - } - prbs_polynomial: PRBS_POLYNOMIAL_PRBS23 - test_duration_in_secs: 180 - } - )PROTO"); - // Set a random BERT operation id. + ASSERT_OK_AND_ASSIGN(std::unique_ptr sut_gnmi_stub, + sut.CreateGnmiStub()); + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sut_gnoi_diag_stub, + sut.CreateGnoiDiagStub()); + + // TODO (b/182417612) : Select one operational state "up" port. + constexpr char kInterface[] = "Ethernet0"; + gnoi::diag::StartBERTRequest bert_request; + // Create the BERT request. bert_request.set_bert_operation_id( absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + *(bert_request.add_per_port_requests()) = + gutil::ParseProtoOrDie( + BuildPerPortStartBertRequest(kInterface)); gnoi::diag::StartBERTResponse bert_response; grpc::ClientContext context; LOG(INFO) << "Sending StartBERT request: " << bert_request.ShortDebugString(); + EXPECT_OK( sut_gnoi_diag_stub->StartBERT(&context, bert_request, &bert_response)); - // Poll for a maximum of 10 minutes to check if BERT is completed. - constexpr absl::Duration kPollInterval = absl::Seconds(30); - constexpr int kMaxPollCount = 20; + // Poll for allowed BERT delay duration. + int max_poll_count = + ceil(ToInt64Seconds(kDelayDuration) / ToInt64Seconds(kPollInterval)); bool poll_timeout = true; - gnoi::diag::GetBERTResultRequest result_request; - result_request.set_bert_operation_id(bert_request.bert_operation_id()); - result_request.add_per_port_requests()->mutable_interface()->CopyFrom( - bert_request.per_port_requests(0).interface()); - for (int count = 0; count < kMaxPollCount; ++count) { + for (int count = 0; count < max_poll_count; ++count) { absl::SleepFor(kPollInterval); - auto statusor = - pins_test::GetInterfaceOperStatusOverGnmi(*sut_gnmi_stub, "Ethernet0"); + ASSERT_OK_AND_ASSIGN( + pins_test::OperStatus oper_status, + pins_test::GetInterfaceOperStatusOverGnmi(*sut_gnmi_stub, kInterface)); // If port is "UP" and no longer in "TESTING" oper state, BERT has completed // on the port and full BERT result is ready for read. - if (statusor.ok() && (*statusor == pins_test::OperStatus::kUp)) { + if (oper_status == pins_test::OperStatus::kUp) { poll_timeout = false; break; } @@ -184,15 +230,230 @@ TEST_P(BertTest, StartBertfailsIfPeerPortNotRunningBert) { LOG(WARNING) << "BERT is not completed on the port in maximum allowed time."; } + + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id(bert_request.bert_operation_id()); + *(result_request.add_per_port_requests()->mutable_interface()) = + bert_request.per_port_requests(0).interface(); + gnoi::diag::GetBERTResultResponse result_response; grpc::ClientContext result_context; EXPECT_OK(sut_gnoi_diag_stub->GetBERTResult(&result_context, result_request, &result_response)); LOG(INFO) << "Result: " << result_response.ShortDebugString(); + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); // Verify the response. ASSERT_THAT(result_response.per_port_responses(), SizeIs(1)); EXPECT_EQ(result_response.per_port_responses(0).status(), gnoi::diag::BERT_STATUS_PEER_LOCK_FAILURE); } +// Since BERT test is a time consuming test, we decided to combine few tests +// together to save BERT test run time. This test runs and verifies following +// cases: +// 1) BERT runs successfully on 2 ports. +// 2) While BERT is running on ports, another attempt to start the BERT on these +// same ports should fail. +// 3) Operation id that was used earlier to start the BERT test will fail to +// start BERT if used again. +TEST_P(BertTest, StartBertSucceeds) { + thinkit::Switch& sut = GetMirrorTestbed().Sut(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr sut_gnmi_stub, + sut.CreateGnmiStub()); + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sut_gnoi_diag_stub, + sut.CreateGnoiDiagStub()); + + thinkit::Switch& control_switch = GetMirrorTestbed().ControlSwitch(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr control_switch_gnmi_stub, + control_switch.CreateGnmiStub()); + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(control_switch)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr control_switch_gnoi_diag_stub, + control_switch.CreateGnoiDiagStub()); + + // TODO (b/182417612) : Select 2 operational state "up" ports. + std::vector interfaces = {"Ethernet0", "Ethernet8"}; + gnoi::diag::StartBERTRequest bert_request; + // Create the BERT request. + bert_request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + for (const std::string& interface : interfaces) { + *(bert_request.add_per_port_requests()) = + gutil::ParseProtoOrDie( + BuildPerPortStartBertRequest(interface)); + } + + // Request StartBert on the SUT switch. + { + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + LOG(INFO) << "Sending StartBERT request: " + << bert_request.ShortDebugString(); + EXPECT_OK( + sut_gnoi_diag_stub->StartBERT(&context, bert_request, &bert_response)); + } + + // Request StartBert on the control switch. + { + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + LOG(INFO) << "Sending StartBERT request: " + << bert_request.ShortDebugString(); + EXPECT_OK(control_switch_gnoi_diag_stub->StartBERT(&context, bert_request, + &bert_response)); + } + + // Wait for sync duration. + absl::SleepFor(kSyncDuration); + // Verify that ports should be in TESTING mode now. + for (const std::string& interface : interfaces) { + SCOPED_TRACE( + absl::StrCat("Getting operational status for interface: ", interface)); + ASSERT_OK_AND_ASSIGN( + pins_test::OperStatus oper_status, + pins_test::GetInterfaceOperStatusOverGnmi(*sut_gnmi_stub, interface)); + ASSERT_TRUE(oper_status == pins_test::OperStatus::kTesting); + ASSERT_OK_AND_ASSIGN(oper_status, + pins_test::GetInterfaceOperStatusOverGnmi( + *control_switch_gnmi_stub, interface)); + ASSERT_TRUE(oper_status == pins_test::OperStatus::kTesting); + } + + // Request another StartBert on the same ports on SUT and it should fail. + { + gnoi::diag::StartBERTRequest second_bert_request = bert_request; + second_bert_request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + LOG(INFO) << "Sending second StartBERT request: " + << second_bert_request.ShortDebugString(); + EXPECT_OK(sut_gnoi_diag_stub->StartBERT(&context, second_bert_request, + &bert_response)); + + // Wait some time before querying the result. + absl::SleepFor(kWaitTime); + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id( + second_bert_request.bert_operation_id()); + for (int idx = 0; idx < interfaces.size(); ++idx) { + *(result_request.add_per_port_requests()->mutable_interface()) = + second_bert_request.per_port_requests(idx).interface(); + } + + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext result_context; + EXPECT_OK(sut_gnoi_diag_stub->GetBERTResult(&result_context, result_request, + &result_response)); + LOG(INFO) << "Result: " << result_response.ShortDebugString(); + EXPECT_THAT(result_response.per_port_responses(), + SizeIs(interfaces.size())); + for (const auto& per_port_result : result_response.per_port_responses()) { + EXPECT_EQ(per_port_result.status(), + gnoi::diag::BERT_STATUS_PORT_ALREADY_IN_BERT) + << "Port result: " << per_port_result.ShortDebugString(); + } + } + + + // Poll for remaining BERT duration. + int max_poll_count = + 1 + (absl::ToInt64Seconds(kDelayDuration + kTestDuration - kSyncDuration - + kWaitTime - absl::Seconds(1)) / + ToInt64Seconds(kPollInterval)); + std::vector interfaces_not_up = interfaces; + for (int count = 0; count < max_poll_count; ++count) { + absl::SleepFor(kPollInterval); + // If port is "UP" and no longer in "TESTING" oper state on both sides of + // link, BERT has completed on the link and full BERT result is ready. + for (auto it = interfaces_not_up.begin(); it != interfaces_not_up.end();) { + ASSERT_OK_AND_ASSIGN( + pins_test::OperStatus oper_status, + pins_test::GetInterfaceOperStatusOverGnmi(*sut_gnmi_stub, *it)); + if (oper_status == pins_test::OperStatus::kUp) { + ASSERT_OK_AND_ASSIGN(oper_status, + pins_test::GetInterfaceOperStatusOverGnmi( + *control_switch_gnmi_stub, *it)); + if (oper_status == pins_test::OperStatus::kUp) { + it = interfaces_not_up.erase(it); + continue; + } + } + ++it; + } + if (interfaces_not_up.empty()) break; + } + + EXPECT_THAT(interfaces_not_up, testing::IsEmpty()); + + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id(bert_request.bert_operation_id()); + for (int idx = 0; idx < interfaces.size(); ++idx) { + *(result_request.add_per_port_requests()->mutable_interface()) = + bert_request.per_port_requests(idx).interface(); + } + // Get the BERT result from the SUT. + + { + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext result_context; + EXPECT_OK(sut_gnoi_diag_stub->GetBERTResult(&result_context, result_request, + &result_response)); + LOG(INFO) << "Result: " << result_response.ShortDebugString(); + ASSERT_THAT(result_response.per_port_responses(), + SizeIs(bert_request.per_port_requests_size())); + for (int idx = 0; idx < result_response.per_port_responses_size(); ++idx) { + VerifyBertResultForSuccess( + result_response.per_port_responses(idx), + bert_request.bert_operation_id(), + bert_request.per_port_requests(idx).interface(), + bert_request.per_port_requests(idx).prbs_polynomial()); + } + } + + // Get the BERT result from the control switch. + { + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext result_context; + EXPECT_OK(control_switch_gnoi_diag_stub->GetBERTResult( + &result_context, result_request, &result_response)); + LOG(INFO) << "Result: " << result_response.ShortDebugString(); + ASSERT_THAT(result_response.per_port_responses(), + SizeIs(bert_request.per_port_requests_size())); + for (int idx = 0; idx < result_response.per_port_responses_size(); ++idx) { + VerifyBertResultForSuccess( + result_response.per_port_responses(idx), + bert_request.bert_operation_id(), + bert_request.per_port_requests(idx).interface(), + bert_request.per_port_requests(idx).prbs_polynomial()); + } + } + + // Request another StartBert on the SUT with just used operation id, it should + // fail. + { + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + LOG(INFO) << "Sending StartBERT request: " + << bert_request.ShortDebugString(); + EXPECT_THAT( + gutil::GrpcStatusToAbslStatus(sut_gnoi_diag_stub->StartBERT( + &context, bert_request, &bert_response)), + gutil::StatusIs(absl::StatusCode::kInvalidArgument, + AllOf(HasSubstr("Invalid request"), + HasSubstr(bert_request.bert_operation_id()), + HasSubstr("exists")))) + << "Response: " << bert_response.ShortDebugString(); + } + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); + // ASSERT_OK(pins_test::PortsUp(control_switch)); +} + } // namespace bert From 459646453d9f11e5757446cf7459ad19abdc419f Mon Sep 17 00:00:00 2001 From: smolkaj Date: Mon, 28 Oct 2024 05:46:39 -0700 Subject: [PATCH 3/4] Adding additional BERT tests. Adding new smoke test --- tests/forwarding/smoke_test.cc | 40 ++++++++++ tests/gnoi/bert_tests.cc | 142 ++++++++++++++++++++++++++++++--- 2 files changed, 171 insertions(+), 11 deletions(-) diff --git a/tests/forwarding/smoke_test.cc b/tests/forwarding/smoke_test.cc index cbefe6a7..308bcc2a 100644 --- a/tests/forwarding/smoke_test.cc +++ b/tests/forwarding/smoke_test.cc @@ -28,6 +28,46 @@ namespace gpins { namespace { +TEST_P(SmokeTestFixture, ModifyWorks) { + const sai::WriteRequest pd_insert = gutil::ParseProtoOrDie( + R"pb( + updates { + type: INSERT + table_entry { + acl_ingress_table_entry { + match { is_ip { value: "0x1" } } + priority: 10 + action { copy { qos_queue: "0x1" } } + } + } + } + )pb"); + ASSERT_OK_AND_ASSIGN(p4::v1::WriteRequest pi_insert, + pdpi::PdWriteRequestToPi(IrP4Info(), pd_insert)); + ASSERT_OK( + pdpi::SetMetadataAndSendPiWriteRequest(SutP4RuntimeSession(), pi_insert)); + + const sai::WriteRequest pd_modify = gutil::ParseProtoOrDie( + R"pb( + updates { + type: MODIFY + table_entry { + acl_ingress_table_entry { + match { is_ip { value: "0x1" } } + priority: 10 + action { forward {} } + } + } + } + )pb"); + ASSERT_OK_AND_ASSIGN(p4::v1::WriteRequest pi_modify, + pdpi::PdWriteRequestToPi(IrP4Info(), pd_modify)); + ASSERT_OK( + pdpi::SetMetadataAndSendPiWriteRequest(SutP4RuntimeSession(), pi_modify)); + // This used to fail with a read error, see b/185508142. + ASSERT_OK(pdpi::ClearTableEntries(SutP4RuntimeSession())); +} + TEST_P(SmokeTestFixture, InsertTableEntry) { const sai::TableEntry pd_entry = gutil::ParseProtoOrDie( R"pb( diff --git a/tests/gnoi/bert_tests.cc b/tests/gnoi/bert_tests.cc index a1ba8b27..7b3327e1 100644 --- a/tests/gnoi/bert_tests.cc +++ b/tests/gnoi/bert_tests.cc @@ -64,6 +64,18 @@ const std::string BuildPerPortStartBertRequest( interface_name, ToInt64Seconds(kTestDuration)); } +const std::string BuildOpenConfigInterface(absl::string_view interface_name) { + return absl::Substitute(R"pb( + origin: "openconfig" + elem { name: "interfaces" } + elem { + name: "interface" + key { key: "name" value: '$0' } + } + )pb", + interface_name); +} + void VerifyBertResultForSuccess( const gnoi::diag::GetBERTResultResponse::PerPortResponse& bert_result, absl::string_view op_id, const gnoi::types::Path& interface, @@ -159,18 +171,10 @@ TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { // Case 4: Invalid interface. { gnoi::diag::StartBERTRequest invalid_interface_request = valid_request; - gnoi::types::Path invalid_interface = - gutil::ParseProtoOrDie( - R"pb( - origin: "openconfig" - elem { name: "interfaces" } - elem { - name: "interface" - key { key: "name" value: "InvalidPort" } - } - )pb"); *(invalid_interface_request.mutable_per_port_requests(0) - ->mutable_interface()) = invalid_interface; + ->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface("InvalidPort")); response.Clear(); grpc::ClientContext context; LOG(INFO) << "Sending StartBERT request: " @@ -184,6 +188,122 @@ TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { // ASSERT_OK(pins_test::PortsUp(sut)); } +// Test StopBERT RPC with invalid argument in the request. +// 1) If StopBERT RPC is requested on an invalid port, RPC should fail. +// 2) If StopBERT RPC is requested on a port that is not running BERT, RPC +// should fail. +TEST_P(BertTest, StopBertfailsIfRequestParametersInvalid) { + thinkit::Switch& sut = GetMirrorTestbed().Sut(); + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sut_gnoi_diag_stub, + sut.CreateGnoiDiagStub()); + + // Request StopBERT RPC on an invalid port, RPC should fail. + { + gnoi::diag::StopBERTRequest request; + request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + *(request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface("invalidPort")); + + gnoi::diag::StopBERTResponse response; + grpc::ClientContext context; + LOG(INFO) << "Sending StopBERT request: " << request.ShortDebugString(); + EXPECT_THAT( + gutil::GrpcStatusToAbslStatus( + sut_gnoi_diag_stub->StopBERT(&context, request, &response)), + gutil::StatusIs( + absl::StatusCode::kInvalidArgument, + AllOf(HasSubstr("Interface is invalid"), + HasSubstr("Operation ID is not found on interface")))); + } + + // Request StopBERT RPC on a port that is not running BERT, RPC should fail. + { + // TODO (b/182417612) : Select one operational state "up" port. + constexpr char kInterface[] = "Ethernet0"; + gnoi::diag::StopBERTRequest request; + request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + *(request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface(kInterface)); + gnoi::diag::StopBERTResponse response; + grpc::ClientContext context; + LOG(INFO) << "Sending StopBERT request: " << request.ShortDebugString(); + EXPECT_THAT( + gutil::GrpcStatusToAbslStatus( + sut_gnoi_diag_stub->StopBERT(&context, request, &response)), + gutil::StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Operation ID is not found on interface"))); + } + + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); +} + +// Test GetBERTResult RPC with invalid argument in the request. +// 1) If GetBERTResult RPC is requested on an invalid port, RPC should fail. +// 2) If GetBERTResult RPC is requested on a port that never ran BERT before, +// RPC should fail. +TEST_P(BertTest, GetBertResultFailsIfRequestParametersInvalid) { + thinkit::Switch& sut = GetMirrorTestbed().Sut(); + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sut_gnoi_diag_stub, + sut.CreateGnoiDiagStub()); + + // Request GetBERTResult RPC on an invalid port, RPC should fail. + { + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + *(result_request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface("InvalidPort")); + + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext context; + LOG(INFO) << "Sending GetBERTResult request: " + << result_request.ShortDebugString(); + EXPECT_THAT(gutil::GrpcStatusToAbslStatus(sut_gnoi_diag_stub->GetBERTResult( + &context, result_request, &result_response)), + gutil::StatusIs( + absl::StatusCode::kInvalidArgument, + AllOf(HasSubstr("Interface"), HasSubstr("is not valid")))); + } + // Request GetBERTResult RPC on a port that never ran BERT before, RPC should + // fail. + { + // TODO (b/182417612) : Select one operational state "up" port. + constexpr char kInterface[] = "Ethernet0"; + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + *(result_request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface(kInterface)); + + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext context; + LOG(INFO) << "Sending GetBERTResult request: " + << result_request.ShortDebugString(); + EXPECT_THAT(gutil::GrpcStatusToAbslStatus(sut_gnoi_diag_stub->GetBERTResult( + &context, result_request, &result_response)), + gutil::StatusIs(absl::StatusCode::kInvalidArgument, + AllOf(HasSubstr("Result is not found for intf"), + HasSubstr(kInterface)))); + } + + // TODO (b/176913347): Enable the all ports UP check. + // ASSERT_OK(pins_test::PortsUp(sut)); +} + + // Test StartBERT fails if peer port is not running BERT. TEST_P(BertTest, StartBertfailsIfPeerPortNotRunningBert) { thinkit::Switch& sut = GetMirrorTestbed().Sut(); From 2175df535710ce7c4c49366bdbea3f4a210fca1c Mon Sep 17 00:00:00 2001 From: smolkaj Date: Tue, 29 Oct 2024 01:05:34 -0700 Subject: [PATCH 4/4] visibility declarations; make all targets public. Set role. Adding additional BERT tests. --- tests/forwarding/BUILD.bazel | 1 - tests/forwarding/p4_blackbox_fixture.h | 11 +- tests/forwarding/smoke_test.cc | 5 +- tests/gnoi/BUILD.bazel | 2 + tests/gnoi/bert_tests.cc | 333 +++++++++++++++++++++++++ 5 files changed, 347 insertions(+), 5 deletions(-) diff --git a/tests/forwarding/BUILD.bazel b/tests/forwarding/BUILD.bazel index 28bf4cdf..ba4e6343 100644 --- a/tests/forwarding/BUILD.bazel +++ b/tests/forwarding/BUILD.bazel @@ -82,7 +82,6 @@ cc_library( testonly = True, srcs = ["smoke_test.cc"], hdrs = ["smoke_test.h"], - visibility = ["//visibility:public"], deps = [ ":p4_blackbox_fixture", ":test_data", diff --git a/tests/forwarding/p4_blackbox_fixture.h b/tests/forwarding/p4_blackbox_fixture.h index a0d63e25..57bbcc82 100644 --- a/tests/forwarding/p4_blackbox_fixture.h +++ b/tests/forwarding/p4_blackbox_fixture.h @@ -68,7 +68,7 @@ class P4BlackboxFixture : public thinkit::MirrorTestbedFixture { } void TearDown() override { - if (SutP4RuntimeSession() != nullptr) { + if (SutP4RuntimeSession() != nullptr && clear_table_entries_on_teardown_) { // Clear all table entries to leave the switch in a clean state. EXPECT_OK(pdpi::ClearTableEntries(SutP4RuntimeSession())); } @@ -82,9 +82,16 @@ class P4BlackboxFixture : public thinkit::MirrorTestbedFixture { const pdpi::IrP4Info& IrP4Info() const { return ir_p4info_; } + protected: + void DisableClearingTableEntriesOnTearDown() { + clear_table_entries_on_teardown_ = false; + } + private: + bool clear_table_entries_on_teardown_ = true; std::unique_ptr sut_p4rt_session_; - pdpi::IrP4Info ir_p4info_ = sai::GetIrP4Info(sai::Instantiation::kMiddleblock); + pdpi::IrP4Info ir_p4info_ = + sai::GetIrP4Info(sai::Instantiation::kMiddleblock); }; } // namespace gpins diff --git a/tests/forwarding/smoke_test.cc b/tests/forwarding/smoke_test.cc index 308bcc2a..d82fc671 100644 --- a/tests/forwarding/smoke_test.cc +++ b/tests/forwarding/smoke_test.cc @@ -129,8 +129,9 @@ TEST_P(SmokeTestFixture, InsertAndReadTableEntries) { p4::v1::ReadRequest read_request; read_request.add_entities()->mutable_table_entry(); - ASSERT_OK_AND_ASSIGN(p4::v1::ReadResponse read_response, - pdpi::SetMetadataAndSendPiReadRequest(session, read_request)); + ASSERT_OK_AND_ASSIGN( + p4::v1::ReadResponse read_response, + pdpi::SetMetadataAndSendPiReadRequest(session, read_request)); for (const auto& entity : read_response.entities()) { ASSERT_OK(test_environment.AppendToTestArtifact( diff --git a/tests/gnoi/BUILD.bazel b/tests/gnoi/BUILD.bazel index 9e72a3de..4413fec0 100644 --- a/tests/gnoi/BUILD.bazel +++ b/tests/gnoi/BUILD.bazel @@ -34,6 +34,8 @@ cc_library( "//thinkit:mirror_testbed_fixture", "@com_github_gnoi//diag:diag_cc_proto", "@com_github_google_glog//:glog", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/flags:flag", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", diff --git a/tests/gnoi/bert_tests.cc b/tests/gnoi/bert_tests.cc index 7b3327e1..def55955 100644 --- a/tests/gnoi/bert_tests.cc +++ b/tests/gnoi/bert_tests.cc @@ -14,6 +14,10 @@ #include "tests/gnoi/bert_tests.h" +#include +#include "absl/container/flat_hash_set.h" +#include "absl/flags/flag.h" + #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/substitute.h" @@ -29,6 +33,9 @@ #include "lib/gnmi/gnmi_helper.h" #include "lib/validator/validator_lib.h" +ABSL_FLAG(uint32_t, idx_seed, static_cast(std::time(nullptr)), + "Seed to randomly generate interface index."); + namespace bert { using ::google::protobuf::util::MessageDifferencer; @@ -47,6 +54,11 @@ constexpr absl::Duration kPollInterval = absl::Seconds(30); // Minimum wait time after the BERT request to read the BERT result. constexpr absl::Duration kWaitTime = absl::Seconds(30); +constexpr uint8_t kMaxAllowedInterfacesToRunBert = 96; + +constexpr char kEnabledFalse[] = "{\"enabled\":false}"; +constexpr char kEnabledTrue[] = "{\"enabled\":true}"; + const std::string BuildPerPortStartBertRequest( absl::string_view interface_name) { return absl::Substitute(R"pb( @@ -76,6 +88,65 @@ const std::string BuildOpenConfigInterface(absl::string_view interface_name) { interface_name); } +absl::StatusOr GetInterfaceNameFromOcInterfacePath( + const gnoi::types::Path& interface) { + // Validate if interface is in valid format(either in OpenConfig format or + // SONiC format). + // Example: openconfig/interfaces/interface[name=Ethernet0] or + // openconfig-interfaces/interfaces/interface[name=Ethernet0]. + std::stringstream error_stream; + if ((interface.origin() != "openconfig") && + (interface.origin() != "openconfig-interfaces")) { + error_stream << "Invalid interface origin. Expected: openconfig or " + "openconfig-interfaces but received: " + << interface.origin() << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + auto elems = interface.elem(); + if (elems.size() != 2) { + error_stream << "Invalid element path count. Expected: 2 but received: " + << elems.size() << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + if ((elems[0].name() != "interfaces") || (elems[0].key_size() > 0)) { + error_stream << "First interface path element is malformed. Expected " + "element name: interfaces but received: " + << elems[0].name() + << " and expected 0 key but received: " << elems[0].key_size() + << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + if ((elems[1].name() != "interface")) { + error_stream << "Second interface path element is malformed. Expected " + "element name: interface but received: " + << elems[1].name() << " in: " << interface.DebugString(); + return absl::InvalidArgumentError(error_stream.str()); + } + auto it = elems[1].key().find("name"); + if (it != elems[1].key().end()) { + return it->second; + } + return absl::InvalidArgumentError( + absl::StrCat("Interface name is missing in: ", interface.DebugString())); +} + +void SetAdminStateOnInterfaceList(gnmi::gNMI::Stub& gnmi_stub, + std::vector& interfaces, + absl::string_view value) { + for (const std::string& interface : interfaces) { + const std::string if_enabled_config_path = absl::StrCat( + "interfaces/interface[name=", interface, "]/config/enabled"); + ASSERT_OK(SetGnmiConfigPath(&gnmi_stub, if_enabled_config_path, + pins_test::GnmiSetType::kUpdate, value)); + } +} + +bool IsInterfaceInList(absl::string_view interface, + std::vector& interfaces) { + return std::find(interfaces.begin(), interfaces.end(), interface) != + interfaces.end(); +} + void VerifyBertResultForSuccess( const gnoi::diag::GetBERTResultResponse::PerPortResponse& bert_result, absl::string_view op_id, const gnoi::types::Path& interface, @@ -104,6 +175,143 @@ void VerifyBertResultForSuccess( EXPECT_EQ(bert_result.total_errors(), total_errors); } +// Helps in getting the BERT result on a set of ports and if BERT is running on +// the port, forces admin status down by disabling it. It also modifies the +// list of ports in request by removing the port that was running BERT. +void CheckRunningBertAndForceAdminDownHelper( + gnoi::diag::Diag::Stub& gnoi_diag_stub, gnmi::gNMI::Stub& gnmi_stub, + gnoi::diag::GetBERTResultRequest* request) { + gnoi::diag::GetBERTResultResponse response; + grpc::ClientContext context; + ASSERT_OK(gnoi_diag_stub.GetBERTResult(&context, *request, &response)); + + ASSERT_THAT(response.per_port_responses(), + SizeIs(request->per_port_requests_size())); + request->clear_per_port_requests(); + for (const auto& result : response.per_port_responses()) { + // Check if BERT is running. + if ((result.status() == gnoi::diag::BERT_STATUS_OK) && + (result.peer_lock_established())) { + ASSERT_OK_AND_ASSIGN( + const std::string interface, + GetInterfaceNameFromOcInterfacePath(result.interface())); + // Disable the admin status. + const std::string if_enabled_config_path = absl::StrCat( + "interfaces/interface[name=", interface, "]/config/enabled"); + ASSERT_OK(SetGnmiConfigPath(&gnmi_stub, if_enabled_config_path, + pins_test::GnmiSetType::kUpdate, + kEnabledFalse)); + } else { + // Get result for interfaces again that didn't start BERT in last poll. + *(request->add_per_port_requests()->mutable_interface()) = + result.interface(); + } + } +} + +// Checks if BERT is running on the ports where we are supposed to force admin +// status DOWN. If BERT is running, force admin status to DOWN on port. +void CheckRunningBertAndForceAdminDown( + gnoi::diag::Diag::Stub& sut_gnoi_diag_stub, + gnoi::diag::Diag::Stub& control_switch_gnoi_diag_stub, + gnmi::gNMI::Stub& sut_gnmi_stub, gnmi::gNMI::Stub& control_switch_gnmi_stub, + absl::string_view op_id, std::vector& sut_interfaces, + std::vector& control_switch_interfaces) { + gnoi::diag::GetBERTResultRequest sut_request; + sut_request.set_bert_operation_id(std::string(op_id)); + for (const std::string& interface : sut_interfaces) { + *(sut_request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface(interface)); + } + + gnoi::diag::GetBERTResultRequest control_switch_request; + control_switch_request.set_bert_operation_id(std::string(op_id)); + for (const std::string& interface : control_switch_interfaces) { + *(control_switch_request.add_per_port_requests()->mutable_interface()) = + gutil::ParseProtoOrDie( + BuildOpenConfigInterface(interface)); + } + + int max_poll_count = + 1 + (absl::ToInt64Seconds(kSyncDuration - absl::Seconds(1)) / + absl::ToInt64Seconds(kPollInterval)); + + while (max_poll_count > 0) { + absl::SleepFor(kPollInterval); + if (sut_request.per_port_requests_size() > 0) { + // Check BERT status on SUT and force admin status down. + ASSERT_NO_FATAL_FAILURE(CheckRunningBertAndForceAdminDownHelper( + sut_gnoi_diag_stub, sut_gnmi_stub, &sut_request)); + } + + if (control_switch_request.per_port_requests_size() > 0) { + // Check BERT status on control switch and force admin status down. + ASSERT_NO_FATAL_FAILURE(CheckRunningBertAndForceAdminDownHelper( + control_switch_gnoi_diag_stub, control_switch_gnmi_stub, + &control_switch_request)); + } + if (sut_request.per_port_requests().empty() && + control_switch_request.per_port_requests().empty()) { + break; + } + --max_poll_count; + } + + EXPECT_THAT(sut_request.per_port_requests(), testing::IsEmpty()); + EXPECT_THAT(control_switch_request.per_port_requests(), testing::IsEmpty()); +} + +// Gets the BERT result on all the ports that are running BERT. Verifies BERT +// failure result on ports where admin status was forced to DOWN. Other ports +// are expected to have successful BERT results. +void GetAndVerifyBertResultsWithAdminDownInterfaces( + gnoi::diag::Diag::Stub& gnoi_diag_stub, + gnoi::diag::StartBERTRequest& bert_request, + std::vector& sut_admin_down_interfaces, + std::vector& control_switch_admin_down_interfaces) { + gnoi::diag::GetBERTResultRequest result_request; + result_request.set_bert_operation_id(bert_request.bert_operation_id()); + for (unsigned idx = 0; idx < bert_request.per_port_requests_size(); ++idx) { + *(result_request.add_per_port_requests()->mutable_interface()) = + bert_request.per_port_requests(idx).interface(); + } + gnoi::diag::GetBERTResultResponse result_response; + grpc::ClientContext result_context; + ASSERT_OK(gnoi_diag_stub.GetBERTResult(&result_context, result_request, + &result_response)); + ASSERT_THAT(result_response.per_port_responses(), + SizeIs(bert_request.per_port_requests_size())); + for (unsigned idx = 0; idx < result_response.per_port_responses_size(); + ++idx) { + // Extract interface name from OC interface path. + ASSERT_OK_AND_ASSIGN( + const std::string interface_name, + GetInterfaceNameFromOcInterfacePath( + result_response.per_port_responses(idx).interface())); + // Check if interface is part of list where admin state was disabled. + if (IsInterfaceInList(interface_name, sut_admin_down_interfaces) || + IsInterfaceInList(interface_name, + control_switch_admin_down_interfaces)) { + // Verify BERT failure. + EXPECT_EQ(result_response.per_port_responses(idx).status(), + gnoi::diag::BERT_STATUS_PEER_LOCK_LOST); + EXPECT_TRUE( + result_response.per_port_responses(idx).peer_lock_established()); + EXPECT_TRUE(result_response.per_port_responses(idx).peer_lock_lost()); + continue; + } + // If it is normal BERT running port, verify normal SUCCESS result. + VerifyBertResultForSuccess( + result_response.per_port_responses(idx), + bert_request.bert_operation_id(), + bert_request.per_port_requests(idx).interface(), + bert_request.per_port_requests(idx).prbs_polynomial()); + } +} + + + // Test StartBERT with invalid request parameters. TEST_P(BertTest, StartBertFailsIfRequestParametersInvalid) { thinkit::Switch& sut = GetMirrorTestbed().Sut(); @@ -576,4 +784,129 @@ TEST_P(BertTest, StartBertSucceeds) { // ASSERT_OK(pins_test::PortsUp(control_switch)); } +// Runs the BERT test on current maximum allowed number of interfaces. During +// the BERT run: +// 1) Disable admin state of few ports on SUT, +// 2) Disable admin state of few ports on control switch, +// This helps us verify a mix of operation during BERT - unexpected software or +// hardware errors. +TEST_P(BertTest, RunBertOnMaximumAllowedPorts) { + thinkit::Switch& sut = GetMirrorTestbed().Sut(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr sut_gnmi_stub, + sut.CreateGnmiStub()); + ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr sut_gnoi_diag_stub, + sut.CreateGnoiDiagStub()); + + thinkit::Switch& control_switch = GetMirrorTestbed().ControlSwitch(); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr control_switch_gnmi_stub, + control_switch.CreateGnmiStub()); + ASSERT_OK(pins_test::PortsUp(control_switch)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr control_switch_gnoi_diag_stub, + control_switch.CreateGnoiDiagStub()); + + // Get all the interfaces that are operational status "UP". + ASSERT_OK_AND_ASSIGN(std::vector interfaces, + pins_test::GetUpInterfacesOverGnmi(*sut_gnmi_stub)); + // For this test, we need at least 5 UP interfaces. + ASSERT_GE(interfaces.size(), 5); + // Resize the interface list if UP ports are more than max number of allowed + // ports. + if (interfaces.size() > kMaxAllowedInterfacesToRunBert) { + interfaces.resize(kMaxAllowedInterfacesToRunBert); + } + + // Select 'N' ports on control switch and 'N' ports on SUT for admin down. + int num_interfaces_to_disable = 1 + (interfaces.size() / 16); + // Seed the std::rand(). + LOG(INFO) << "Seeding pseudo-random number generator with seed: " + << absl::GetFlag(FLAGS_idx_seed); + // Select SUT interfaces in the range [0..interfaces/2). + std::vector sut_interfaces_for_admin_down; + std::sample(interfaces.begin(), interfaces.begin() + interfaces.size() / 2, + std::back_inserter(sut_interfaces_for_admin_down), + num_interfaces_to_disable, + std::mt19937(absl::GetFlag(FLAGS_idx_seed))); + // Select control switch interfaces in the range [interfaces/2..interfaces]. + std::vector control_switch_interfaces_for_admin_down; + std::sample(interfaces.begin() + interfaces.size() / 2, interfaces.end(), + std::back_inserter(control_switch_interfaces_for_admin_down), + num_interfaces_to_disable, + std::mt19937(absl::GetFlag(FLAGS_idx_seed))); + + LOG(INFO) << "Starting BERT on " << interfaces.size() << " interfaces."; + + gnoi::diag::StartBERTRequest bert_request; + // Create the BERT request. + bert_request.set_bert_operation_id( + absl::StrCat("OpId-", absl::ToUnixMillis(absl::Now()))); + for (const std::string& interface : interfaces) { + *(bert_request.add_per_port_requests()) = + gutil::ParseProtoOrDie( + BuildPerPortStartBertRequest(interface)); + } + + // Request StartBert on the SUT switch. + { + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + EXPECT_OK( + sut_gnoi_diag_stub->StartBERT(&context, bert_request, &bert_response)); + } + + // Request StartBert on the control switch. + { + gnoi::diag::StartBERTResponse bert_response; + grpc::ClientContext context; + EXPECT_OK(control_switch_gnoi_diag_stub->StartBERT(&context, bert_request, + &bert_response)); + } + + absl::Time start_time = absl::Now(); + // Give some time to BERT to move in SYNC state. + absl::SleepFor(absl::Seconds(90)); + + absl::Time end_time = absl::Now(); + + // Poll for remaining BERT duration. + int max_poll_count = + 1 + (absl::ToInt64Seconds(kDelayDuration + kWaitTime + kSyncDuration + + kTestDuration - (end_time - start_time) - + absl::Seconds(1)) / + ToInt64Seconds(kPollInterval)); + std::vector interfaces_not_up = interfaces; + for (int count = 0; count < max_poll_count; ++count) { + absl::SleepFor(kPollInterval); + // If port is "UP" and no longer in "TESTING" oper state on both sides of + // link, BERT has completed on the link and full BERT result is ready. + for (auto it = interfaces_not_up.begin(); it != interfaces_not_up.end();) { + ASSERT_OK_AND_ASSIGN( + pins_test::OperStatus oper_status, + pins_test::GetInterfaceOperStatusOverGnmi(*sut_gnmi_stub, *it)); + if (oper_status == pins_test::OperStatus::kUp) { + ASSERT_OK_AND_ASSIGN(oper_status, + pins_test::GetInterfaceOperStatusOverGnmi( + *control_switch_gnmi_stub, *it)); + if (oper_status == pins_test::OperStatus::kUp) { + it = interfaces_not_up.erase(it); + continue; + } + } + ++it; + } + if (interfaces_not_up.empty()) break; + } + + EXPECT_THAT(interfaces_not_up, testing::IsEmpty()); + + // Wait for some time before checking the port status. + absl::SleepFor(absl::Seconds(10)); + + ASSERT_OK(pins_test::PortsUp(sut)); + ASSERT_OK(pins_test::PortsUp(control_switch)); +} + } // namespace bert