From 0a7bb2c4c185da115aab61ef000eb274859cf72e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=97=AD?= Date: Tue, 21 Nov 2023 16:11:02 +0800 Subject: [PATCH] Fix ORC related failed UT --- velox/connectors/hive/HiveConnector.cpp | 1 + velox/dwio/dwrf/common/FileMetadata.h | 4 ++-- velox/dwio/dwrf/reader/DwrfReader.cpp | 8 ++++++++ velox/dwio/dwrf/reader/DwrfReader.h | 15 +++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/velox/connectors/hive/HiveConnector.cpp b/velox/connectors/hive/HiveConnector.cpp index cd1344c19e153..fdcacb5b61955 100644 --- a/velox/connectors/hive/HiveConnector.cpp +++ b/velox/connectors/hive/HiveConnector.cpp @@ -150,6 +150,7 @@ std::unique_ptr HivePartitionFunctionSpec::create( void HiveConnectorFactory::initialize() { static bool once = []() { dwio::common::registerFileSinks(); + dwrf::registerOrcReaderFactory(); dwrf::registerDwrfReaderFactory(); dwrf::registerDwrfWriterFactory(); // Meta's buck build system needs this check. diff --git a/velox/dwio/dwrf/common/FileMetadata.h b/velox/dwio/dwrf/common/FileMetadata.h index 2ea21628a595a..973ec312ecd63 100644 --- a/velox/dwio/dwrf/common/FileMetadata.h +++ b/velox/dwio/dwrf/common/FileMetadata.h @@ -426,7 +426,8 @@ class FooterWrapper : public ProtoWrapperBase { // TODO: ORC has not supported column statistics yet int statisticsSize() const { - return format_ == DwrfFormat::kDwrf ? dwrfPtr()->statistics_size() : 0; + return format_ == DwrfFormat::kDwrf ? dwrfPtr()->statistics_size() + : orcPtr()->statistics_size(); } const ::google::protobuf::RepeatedPtrField< @@ -438,7 +439,6 @@ class FooterWrapper : public ProtoWrapperBase { const ::facebook::velox::dwrf::proto::ColumnStatistics& statistics( int index) const { - VELOX_CHECK_EQ(format_, DwrfFormat::kDwrf); return dwrfPtr()->statistics(index); } diff --git a/velox/dwio/dwrf/reader/DwrfReader.cpp b/velox/dwio/dwrf/reader/DwrfReader.cpp index 98e711e00c5e1..4427b0851f4f1 100644 --- a/velox/dwio/dwrf/reader/DwrfReader.cpp +++ b/velox/dwio/dwrf/reader/DwrfReader.cpp @@ -1108,4 +1108,12 @@ void unregisterDwrfReaderFactory() { dwio::common::unregisterReaderFactory(dwio::common::FileFormat::DWRF); } +void registerOrcReaderFactory() { + dwio::common::registerReaderFactory(std::make_shared()); +} + +void unregisterOrcReaderFactory() { + dwio::common::unregisterReaderFactory(dwio::common::FileFormat::ORC); +} + } // namespace facebook::velox::dwrf diff --git a/velox/dwio/dwrf/reader/DwrfReader.h b/velox/dwio/dwrf/reader/DwrfReader.h index 9ced24322a23f..9c8e32df2c00d 100644 --- a/velox/dwio/dwrf/reader/DwrfReader.h +++ b/velox/dwio/dwrf/reader/DwrfReader.h @@ -365,8 +365,23 @@ class DwrfReaderFactory : public dwio::common::ReaderFactory { } }; +class OrcReaderFactory : public dwio::common::ReaderFactory { + public: + OrcReaderFactory() : ReaderFactory(dwio::common::FileFormat::ORC) {} + + std::unique_ptr createReader( + std::unique_ptr input, + const dwio::common::ReaderOptions& options) override { + return DwrfReader::create(std::move(input), options); + } +}; + void registerDwrfReaderFactory(); void unregisterDwrfReaderFactory(); +void registerOrcReaderFactory(); + +void unregisterOrcReaderFactory(); + } // namespace facebook::velox::dwrf