From 3a3b81c93213fe8ddba057de6d584c30ba177811 Mon Sep 17 00:00:00 2001 From: Jia Ke Date: Fri, 8 Nov 2024 20:38:00 +0800 Subject: [PATCH] Test jvm libhdfs perf --- velox/CMakeLists.txt | 1 - .../hdfs/ArrowHdfsInternal.cpp | 0 .../hdfs/ArrowHdfsInternal.h | 0 .../hive/storage_adapters/hdfs/CMakeLists.txt | 4 ++-- .../storage_adapters/hdfs/HdfsFileSystem.cpp | 2 +- .../storage_adapters/hdfs/HdfsReadFile.cpp | 2 +- .../hive/storage_adapters/hdfs/HdfsReadFile.h | 2 +- .../storage_adapters/hdfs/HdfsWriteFile.cpp | 2 +- .../storage_adapters/hdfs/HdfsWriteFile.h | 2 +- .../hive/storage_adapters}/hdfs/hdfs.h | 0 .../hdfs/tests/HdfsFileSystemTest.cpp | 2 +- velox/external/hdfs/CMakeLists.txt | 19 ------------------- 12 files changed, 8 insertions(+), 28 deletions(-) rename velox/{external => connectors/hive/storage_adapters}/hdfs/ArrowHdfsInternal.cpp (100%) rename velox/{external => connectors/hive/storage_adapters}/hdfs/ArrowHdfsInternal.h (100%) rename velox/{external => connectors/hive/storage_adapters}/hdfs/hdfs.h (100%) delete mode 100644 velox/external/hdfs/CMakeLists.txt diff --git a/velox/CMakeLists.txt b/velox/CMakeLists.txt index 06ae8bf1c053..00c969ccce7b 100644 --- a/velox/CMakeLists.txt +++ b/velox/CMakeLists.txt @@ -24,7 +24,6 @@ add_subdirectory(row) add_subdirectory(flag_definitions) add_subdirectory(external/date) add_subdirectory(external/md5) -add_subdirectory(external/hdfs) # # examples depend on expression diff --git a/velox/external/hdfs/ArrowHdfsInternal.cpp b/velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.cpp similarity index 100% rename from velox/external/hdfs/ArrowHdfsInternal.cpp rename to velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.cpp diff --git a/velox/external/hdfs/ArrowHdfsInternal.h b/velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.h similarity index 100% rename from velox/external/hdfs/ArrowHdfsInternal.h rename to velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.h diff --git a/velox/connectors/hive/storage_adapters/hdfs/CMakeLists.txt b/velox/connectors/hive/storage_adapters/hdfs/CMakeLists.txt index 44aa7be3489c..2b613146553a 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/CMakeLists.txt +++ b/velox/connectors/hive/storage_adapters/hdfs/CMakeLists.txt @@ -22,10 +22,10 @@ if(VELOX_ENABLE_HDFS) PRIVATE HdfsFileSystem.cpp HdfsReadFile.cpp - HdfsWriteFile.cpp) + HdfsWriteFile.cpp + ArrowHdfsInternal.cpp) velox_link_libraries( velox_hdfs - velox_external_hdfs velox_dwio_common Folly::folly xsimd) diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.cpp b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.cpp index 9d3f6e30a67a..c84c1aaca6ec 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.cpp +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.cpp @@ -18,7 +18,7 @@ #include "velox/common/config/Config.h" #include "velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.h" #include "velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.h" -#include "velox/external/hdfs/ArrowHdfsInternal.h" +#include "velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.h" namespace facebook::velox::filesystems { std::string_view HdfsFileSystem::kScheme("hdfs://"); diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.cpp b/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.cpp index d48dd373d344..0d4a00b51249 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.cpp +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.cpp @@ -16,7 +16,7 @@ #include "HdfsReadFile.h" #include -#include "velox/external/hdfs/ArrowHdfsInternal.h" +#include "velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.h" namespace facebook::velox { diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.h b/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.h index b63c2dd933dd..0b8e116352af 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.h +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsReadFile.h @@ -15,7 +15,7 @@ */ #include "velox/common/file/File.h" -#include "velox/external/hdfs/hdfs.h" +#include "velox/connectors/hive/storage_adapters/hdfs/hdfs.h" namespace facebook::velox { diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.cpp b/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.cpp index be668a3133e1..780815f5f9d4 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.cpp +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.cpp @@ -15,7 +15,7 @@ */ #include "velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.h" -#include "velox/external/hdfs/ArrowHdfsInternal.h" +#include "velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.h" namespace facebook::velox { HdfsWriteFile::HdfsWriteFile( diff --git a/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.h b/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.h index fb311b1a6c3d..48aa40c161f8 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.h +++ b/velox/connectors/hive/storage_adapters/hdfs/HdfsWriteFile.h @@ -16,7 +16,7 @@ #pragma once #include "velox/common/file/File.h" -#include "velox/external/hdfs/hdfs.h" +#include "velox/connectors/hive/storage_adapters/hdfs/hdfs.h" namespace facebook::velox { diff --git a/velox/external/hdfs/hdfs.h b/velox/connectors/hive/storage_adapters/hdfs/hdfs.h similarity index 100% rename from velox/external/hdfs/hdfs.h rename to velox/connectors/hive/storage_adapters/hdfs/hdfs.h diff --git a/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsFileSystemTest.cpp b/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsFileSystemTest.cpp index 33dcac02393d..fccebc0a7fd9 100644 --- a/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsFileSystemTest.cpp +++ b/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsFileSystemTest.cpp @@ -26,7 +26,7 @@ #include "velox/connectors/hive/storage_adapters/hdfs/RegisterHdfsFileSystem.h" #include "velox/core/QueryConfig.h" #include "velox/exec/tests/utils/TempFilePath.h" -#include "velox/external/hdfs/ArrowHdfsInternal.h" +#include "velox/connectors/hive/storage_adapters/hdfs/ArrowHdfsInternal.h" #include diff --git a/velox/external/hdfs/CMakeLists.txt b/velox/external/hdfs/CMakeLists.txt deleted file mode 100644 index 0170fb1a6c26..000000000000 --- a/velox/external/hdfs/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if(${VELOX_ENABLE_HDFS}) - velox_add_library(velox_external_hdfs ArrowHdfsInternal.cpp) - velox_link_libraries( - velox_external_hdfs - PUBLIC - arrow) -endif()