From ed80ab863a4ecaa6a96f2c80637136e8955b675a Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Thu, 6 Jul 2023 18:24:19 +0800 Subject: [PATCH 01/40] flink-log4j --- .../launch/FlinkEngineConnLaunchBuilder.scala | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala index cbcae4ea54..2452799ced 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala @@ -24,12 +24,12 @@ import org.apache.linkis.hadoop.common.conf.HadoopConf import org.apache.linkis.manager.common.protocol.bml.BmlResource import org.apache.linkis.manager.engineplugin.common.conf.EnvConfiguration import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest -import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.{variable, USER} -import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder +import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.{USER,PWD, variable} +import org.apache.linkis.manager.engineplugin.common.launch.process.{Environment, JavaProcessEngineConnLaunchBuilder} +import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants.{CLASS_PATH_SEPARATOR, addPathToClassPath} import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel import java.util - import scala.collection.JavaConverters._ class FlinkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { @@ -81,6 +81,18 @@ class FlinkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { } bmlResources } + override def getEnvironment(implicit + engineConnBuildRequest: EngineConnBuildRequest + ): util.Map[String, String] = { + val environment = new util.HashMap[String, String] + addPathToClassPath(environment, variable(PWD)) + val linkisEnvironment = super.getEnvironment + val linkisClassPath = linkisEnvironment.get(Environment.CLASSPATH.toString) + val v = environment.get(Environment.CLASSPATH.toString) + CLASS_PATH_SEPARATOR + linkisClassPath + environment.put(Environment.CLASSPATH.toString, v) + logger.info(environment.asScala.map(e => s"${e._1}->${e._2}").mkString(",")) + environment + } private def contentToBmlResource(userName: String, content: String): BmlResource = { val contentMap = JsonUtils.jackson.readValue(content, classOf[util.Map[String, Object]]) From 37c09705beded02c20131cf57b463e384a64ab0f Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Thu, 3 Aug 2023 18:38:09 +0800 Subject: [PATCH 02/40] edit public --- .../linkis/errorcode/client/handler/LinkisErrorCodeHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java index a5c8d638c7..aa444ac55d 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java @@ -73,7 +73,7 @@ public static LinkisErrorCodeHandler getInstance() { return linkisErrorCodeHandler; } - private LinkisErrorCodeHandler() {} + public LinkisErrorCodeHandler() {} static { // Initialize our timing thread and other thread pools through the getInstance method. From 143fc044c24980ce68c6ecbf828b94cdb1e8dce0 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 17 Sep 2024 23:24:20 +0800 Subject: [PATCH 03/40] merge code --- .../org/apache/linkis/storage/FSFactory.java | 96 -- .../storage/conf/LinkisStorageConf.java | 82 -- .../linkis/storage/csv/StorageCSVWriter.java | 146 --- .../linkis/storage/domain/DataType.java | 237 ----- .../apache/linkis/storage/domain/Dolphin.java | 141 --- .../linkis/storage/domain/MethodEntity.java | 132 --- .../domain/MethodEntitySerializer.java | 84 -- .../LinkisStorageErrorCodeSummary.java | 8 + .../linkis/storage/excel/ExcelXlsReader.java | 278 ----- .../linkis/storage/excel/FirstRowDeal.java | 55 - .../linkis/storage/excel/RowToCsvDeal.java | 66 -- .../storage/excel/StorageExcelWriter.java | 310 ------ .../apache/linkis/storage/excel/XlsUtils.java | 51 +- .../linkis/storage/excel/XlsxUtils.java | 55 +- .../exception/ColLengthExceedException.java | 16 +- .../exception/ColumnIndexExceedException.java | 34 + .../storage/exception/StorageErrorCode.java | 2 - .../factory/impl/BuildHDFSFileSystem.java | 6 +- .../factory/impl/BuildLocalFileSystem.java | 8 +- .../storage/factory/impl/BuildOSSSystem.java | 70 -- .../factory/impl/BuildS3FileSystem.java | 62 -- .../apache/linkis/storage/fs/FileSystem.java | 6 +- .../storage/fs/impl/HDFSFileSystem.java | 98 +- .../storage/fs/impl/LocalFileSystem.java | 43 +- .../linkis/storage/fs/impl/OSSFileSystem.java | 398 -------- .../linkis/storage/fs/impl/S3FileSystem.java | 464 --------- .../io/IOMethodInterceptorFactory.java | 49 - .../resultset/DefaultResultSetFactory.java | 198 ---- .../storage/resultset/OrcResultSetReader.java | 212 ---- .../storage/resultset/OrcResultSetWriter.java | 141 --- .../resultset/ParquetResultSetReader.java | 158 --- .../resultset/ParquetResultSetWriter.java | 135 --- .../storage/resultset/ResultSetFactory.java | 74 -- .../resultset/ResultSetReaderFactory.java | 133 --- .../resultset/ResultSetWriterFactory.java | 102 -- .../storage/resultset/StorageResultSet.java | 88 -- .../resultset/StorageResultSetReader.java | 175 ---- .../resultset/StorageResultSetWriter.java | 268 ----- .../storage/resultset/html/HtmlResultSet.java | 48 - .../resultset/io/IOResultDeserializer.java | 37 - .../resultset/io/IOResultSerializer.java | 47 - .../storage/resultset/io/IOResultSet.java | 43 - .../resultset/picture/PictureResultSet.java | 48 - .../table/TableResultDeserializer.java | 100 -- .../table/TableResultSerializer.java | 111 -- .../resultset/table/TableResultSet.java | 44 - .../resultset/txt/TextResultDeserializer.java | 36 - .../resultset/txt/TextResultSerializer.java | 50 - .../storage/resultset/txt/TextResultSet.java | 46 - .../linkis/storage/script/Compaction.java | 44 - .../linkis/storage/script/ParserFactory.java | 31 - .../linkis/storage/script/ScriptFsWriter.java | 37 - .../linkis/storage/script/Variable.java | 48 - .../linkis/storage/script/VariableParser.java | 131 --- .../script/compaction/PYScriptCompaction.java | 45 - .../script/compaction/QLScriptCompaction.java | 45 - .../compaction/ScalaScriptCompaction.java | 45 - .../compaction/ShellScriptCompaction.java | 45 - .../script/parser/CommonScriptParser.java | 75 -- .../storage/script/parser/PYScriptParser.java | 45 - .../storage/script/parser/QLScriptParser.java | 45 - .../script/parser/ScalaScriptParser.java | 45 - .../script/parser/ShellScriptParser.java | 45 - .../script/reader/StorageScriptFsReader.java | 161 --- .../script/writer/StorageScriptFsWriter.java | 124 --- .../storage/source/AbstractFileSource.java | 124 --- .../linkis/storage/source/FileSource.java | 181 ---- .../linkis/storage/source/FileSplit.java | 324 ------ .../storage/source/ResultsetFileSource.java | 64 -- .../linkis/storage/source/TextFileSource.java | 71 -- .../linkis/storage/utils/FileSystemUtils.java | 191 ---- .../apache/linkis/storage/utils/OrcUtils.java | 152 --- .../storage/utils/StorageConfiguration.java | 161 --- .../linkis/storage/utils/StorageHelper.java | 157 --- .../linkis/storage/utils/StorageUtils.java | 279 ----- .../org/apache/linkis/storage/FSFactory.scala | 98 ++ .../apache/linkis/storage/LineMetaData.scala | 20 +- .../apache/linkis/storage/LineRecord.scala | 19 +- .../storage/conf/LinkisStorageConf.scala | 81 ++ .../linkis/storage/csv/CSVFsReader.scala | 11 +- .../linkis/storage/csv/CSVFsWriter.scala | 48 + .../linkis/storage/csv/StorageCSVReader.scala | 4 +- .../linkis/storage/csv/StorageCSVWriter.scala | 117 +++ .../linkis/storage/domain/DataType.scala | 164 +++ .../linkis/storage/domain/Dolphin.scala | 130 +++ .../linkis/storage/domain/MethodEntity.scala | 97 ++ .../linkis/storage/excel/ExcelFsReader.scala | 22 + .../linkis/storage/excel/ExcelFsWriter.scala} | 40 +- .../linkis/storage/excel/ExcelXlsReader.java | 378 +++++++ .../storage/excel/StorageExcelReader.scala | 4 +- .../storage/excel/StorageExcelWriter.scala | 236 +++++ .../excel/StorageMultiExcelWriter.scala} | 43 +- .../apache/linkis/storage/io/IOClient.scala | 75 ++ .../io/IOMethodInterceptorCreator.scala} | 46 +- .../resultset/DefaultResultSetFactory.scala | 130 +++ .../storage/resultset/ResultMetaData.scala} | 6 +- .../storage/resultset/ResultRecord.scala | 22 + .../storage/resultset/ResultSetFactory.scala | 69 ++ .../storage/resultset/ResultSetReader.scala | 93 ++ .../storage/resultset/ResultSetWriter.scala | 74 ++ .../storage/resultset/StorageResultSet.scala | 51 + .../resultset/StorageResultSetReader.scala | 174 ++++ .../resultset/StorageResultSetWriter.scala | 230 +++++ .../resultset/html/HtmlResultSet.scala | 34 + .../storage/resultset/io/IOMetaData.scala | 25 + .../storage/resultset/io/IORecord.scala | 25 + .../resultset/io/IOResultDeserializer.scala} | 26 +- .../resultset/io/IOResultSerializer.scala | 43 + .../storage/resultset/io/IOResultSet.scala | 32 + .../resultset/picture/PictureResultSet.scala | 22 +- .../resultset/table/TableMetaData.scala | 20 +- .../storage/resultset/table/TableRecord.scala | 14 +- .../table/TableResultDeserializer.scala | 156 +++ .../table/TableResultSerializer.scala | 99 ++ .../resultset/table/TableResultSet.scala} | 28 +- .../txt/TextResultDeserializer.scala | 34 + .../resultset/txt/TextResultSerializer.scala | 46 + .../storage/resultset/txt/TextResultSet.scala | 33 + .../storage/script/ScriptFsReader.scala | 37 + .../storage/script/ScriptFsWriter.scala | 115 +++ .../storage/script/VariableParser.scala | 117 +++ .../compaction/CommonScriptCompaction.scala} | 30 +- .../compaction/PYScriptCompaction.scala | 39 + .../compaction/QLScriptCompaction.scala | 38 + .../compaction/ScalaScriptCompaction.scala | 38 + .../compaction/ShellScriptCompaction.scala} | 30 +- .../script/parser/CommonScriptParser.scala | 63 ++ .../script/parser/PYScriptParser.scala | 26 +- .../script/parser/QLScriptParser.scala | 26 +- .../script/parser/ScalaScriptParser.scala | 39 + .../script/parser/ShellScriptParser.scala | 39 + .../script/reader/StorageScriptFsReader.scala | 126 +++ .../script/writer/StorageScriptFsWriter.scala | 120 +++ .../storage/source/AbstractFileSource.scala | 82 ++ .../linkis/storage/source/FileSource.scala | 160 +++ .../linkis/storage/source/FileSplit.scala | 202 ++++ .../storage/source/ResultsetFileSource.scala | 53 + .../storage/source/TextFileSource.scala | 52 + .../storage/utils/FileSystemUtils.scala | 171 ++++ .../storage/utils/StorageConfiguration.scala | 85 ++ .../linkis/storage/utils/StorageHelper.scala | 107 ++ .../linkis/storage/utils/StorageUtils.scala | 231 +++++ .../linkis/storage/LineMetaDataTest.java | 36 - .../storage/excel/ExcelStorageReaderTest.java | 206 ++++ .../StorageResultSetWriterFactoryTest.java | 53 - .../writer/StorageScriptFsWriterTest.java | 8 +- .../source/ResultsetFileSourceTest.java | 50 - .../test/resources/result-read-test.dolphin | 1 - .../test/resources/storage-read-test.dolphin | 1 - .../linkis/storage/domain/DataTypeTest.scala | 17 +- .../utils/StorageConfigurationTest.scala | 7 +- .../storage/utils/StorageUtilsTest.scala | 16 +- .../governance/common/utils/ECPathUtils.java | 53 - ...onConf.scala => GovernaceCommonConf.scala} | 21 +- .../constant/job/JobRequestConstants.scala | 5 - .../exception/GovernanceErrorException.scala | 5 +- .../common/protocol/task/RequestTask.scala | 3 +- .../protocol/task/ResponseEngineConnPid.scala | 3 +- .../protocol/task/ResponseTaskExecute.scala | 27 - .../governance/common/utils/ECPathUtils.scala | 66 ++ .../common/utils/EngineConnArguments.scala | 8 +- .../common/utils/GovernanceUtils.scala | 27 +- .../utils/OnceExecutorContentUtils.scala | 10 +- .../core/launch/ProcessEngineConnLaunch.scala | 26 +- .../linkis/ecm/core/utils/PortUtils.scala | 2 +- .../src/main/assembly/distribution.xml | 3 - .../linkis/ecm/restful/ECMRestfulApi.java | 2 +- .../linkis/ecm/scheduled/EcmClearTask.java | 2 +- .../operator/EngineConnLogOperator.java | 219 ---- .../operator/EngineConnYarnLogOperator.java | 149 --- .../impl/DefaultEngineConnKillService.java | 6 +- .../linkis/ecm/utils/ECMCacheUtils.java} | 37 +- .../ecm/server/conf/ECMConfiguration.scala | 17 +- .../server/exception/ECMErrorException.scala | 11 +- .../ecm/server/hook/JarUDFLoadECMHook.scala | 1 + .../operator/EngineConnLogOperator.scala | 193 ++++ .../operator/EngineConnYarnLogOperator.scala | 194 ++++ .../server/service/ECMMetricsService.scala | 4 +- .../AbstractEngineConnLaunchService.scala | 7 +- .../impl/BmlResourceLocalizationService.scala | 2 + .../impl/DefaultECMMetricsService.scala | 25 + .../impl/DefaultECMRegisterService.scala | 22 +- .../service/impl/DefaultOperateService.scala | 24 +- .../impl/ProcessEngineConnLaunchService.scala | 41 +- .../impl/DefaultECMRegisterServiceTest.java | 55 - .../OperableOnceEngineConnOperator.java | 92 -- .../once/executor/OnceExecutor.scala | 8 +- .../OnceExecutorExecutionContext.scala | 8 +- .../OperableOnceEngineConnOperator.scala | 72 ++ .../AsyncConcurrentComputationExecutor.scala | 66 +- .../bml/BmlEnginePreExecuteHook.scala | 4 + .../conf/ComputationExecutorConf.scala | 8 +- .../executor/cs/CSTableRegister.scala | 2 +- .../executor/cs/CSTableResultSetWriter.scala | 2 +- .../execute/ComputationExecutor.scala | 43 +- .../ConcurrentComputationExecutor.scala | 51 +- .../execute/EngineExecutionContext.scala | 37 +- .../hook/ComputationExecutorHook.scala | 7 + .../executor/hook/InitSQLHook.scala | 3 +- .../executor/hook/PythonModuleLoad.scala | 161 +++ .../hook/PythonModuleLoadEngineConnHook.scala | 64 ++ .../executor/hook/PythonSparkEngineHook.scala | 45 + .../executor/hook/UDFLoadEngineConnHook.scala | 3 +- .../executor/hook/UseDatabaseEngineHook.scala | 2 +- .../hook/executor/ExecuteOnceHook.scala | 8 + .../executor/rs/RsOutputStream.scala | 6 +- .../service/TaskExecutionServiceImpl.scala | 255 ++--- .../upstream/ECTaskEntranceMonitor.scala | 2 +- ...ingleThreadUpstreamConnectionMonitor.scala | 21 + .../ECTaskEntranceMonitorService.scala | 3 +- .../utlis/ComputationEngineConstant.scala | 2 + .../common/conf/EngineConnConf.scala | 4 - .../common/conf/EngineConnConstant.scala | 2 +- .../core/util/EngineConnUtils.scala | 2 +- .../executor/log/AbstractLogCache.java | 39 +- .../acessible/executor/log/MountLogCache.java | 2 +- .../acessible/executor/log/SendAppender.java | 38 +- .../acessible/executor/log/TimeLogCache.java | 25 +- .../EngineConnApplicationInfoOperator.java | 57 -- .../AccessibleEngineConnExecution.scala | 2 +- .../info/NodeHealthyInfoManager.scala | 2 +- .../acessible/executor/log/LogHelper.scala | 7 + .../EngineConnApplicationInfoOperator.scala | 52 + .../service/DefaultManagerService.scala | 3 +- .../service/DefaultOperateService.scala | 29 +- .../service/EngineConnTimedLockService.scala | 12 +- .../executor/service/LockService.scala | 7 +- .../utils/AccessableExecutorUtils.scala | 1 + .../hook/CallbackEngineConnHook.scala | 25 +- .../callback/service/EngineConnCallback.scala | 2 +- ...ala => EngineConnIdentifierCallback.scala} | 1 + .../service/EngineConnTimedCallback.scala | 20 + .../executor/ExecutorExecutionContext.scala | 17 +- .../EngineConnExecutorConfiguration.scala | 3 + .../EngineconnCoreErrorCodeSummary.java | 6 +- .../exception/EngineConnPluginErrorCode.scala | 26 + .../EngineConnPluginErrorException.scala | 7 +- .../common/resource/UserNodeResource.scala | 22 + .../linkis/entrance/parser/ParserUtils.java | 2 +- .../entrance/restful/EntranceRestfulApi.java | 2 +- .../linkis-application-manager/pom.xml | 6 + .../loader/EngineConnPluginLoaderConf.java | 44 - .../CacheablesEngineConnPluginLoader.java | 3 +- .../DefaultEngineConnPluginLoader.java | 27 +- .../BmlEngineConnPluginResourceLoader.java | 4 +- .../loader/utils/EngineConnPluginUtils.java | 2 +- .../conf/EngineConnPluginConfiguration.java | 42 - .../EngineConnPluginsLoaderFactory.java | 31 - ...bstractEngineConnBmlResourceGenerator.java | 134 --- ...DefaultEngineConnBmlResourceGenerator.java | 114 --- .../EngineConnLocalizeResourceImpl.java | 69 -- .../DefaultEngineConnLaunchService.java | 106 -- ...faultEngineConnResourceFactoryService.java | 80 -- .../DefaultEngineConnResourceService.java | 351 ------- .../service/EngineConnResourceRequest.java | 53 - .../service/EngineConnResourceService.java | 43 - .../service/GetEngineConnResourceRequest.java | 25 - .../RefreshAllEngineConnResourceRequest.java | 28 - .../RefreshEngineConnResourceRequest.java | 25 - .../impl/EnginePluginAdminServiceImpl.java | 18 +- .../manager/LinkisManagerApplication.java | 1 + .../manager/am/conf/AMConfiguration.java | 134 ++- ...ApplicationManagerSpringConfiguration.java | 32 - .../am/conf/ConfigurationMapCache.java | 4 +- ...DefaultEngineConnConfigurationService.java | 5 +- .../manager/am/conf/ManagerMonitorConf.java | 9 +- .../am/converter/DefaultMetricsConverter.java | 17 +- .../converter}/MetricsConverter.java | 2 +- .../manager/am/exception/AMErrorCode.java | 3 +- .../am/exception/AMErrorException.java | 10 +- .../manager/am/label/AMLabelChecker.java | 2 - .../manager/am/label/AMLabelFilter.java | 4 +- .../common => am}/label/LabelChecker.java | 2 +- .../MultiUserEngineReuseLabelChooser.java | 2 - .../am/locker/DefaultEngineNodeLocker.java | 10 +- .../am/manager/DefaultEMNodeManager.java | 10 +- .../am/manager/DefaultEngineNodeManager.java | 129 +-- .../manager/am/manager/EMNodeManager.java | 7 +- .../manager/am/manager/EngineNodeManager.java | 2 + .../am/pointer/AbstractNodePointer.java | 18 +- .../am/pointer/DefaultEMNodPointer.java | 42 +- .../am/pointer/DefaultEngineNodPointer.java | 10 +- .../am/pointer/DefaultNodePointerBuilder.java | 3 - .../common => am}/pointer/EMNodPointer.java | 2 +- .../pointer/EngineNodePointer.java | 2 +- .../common => am}/pointer/NodePointer.java | 2 +- .../pointer/NodePointerBuilder.java | 2 +- .../am/restful/ECResourceInfoRestfulApi.java | 110 +- .../manager/am/restful/EMRestfulApi.java | 170 +++- .../manager/am/restful/EngineRestfulApi.java | 109 +- .../am/selector/DefaultECAvailableRule.java | 43 - .../am/selector/DefaultNodeSelector.java | 75 -- .../rule/AvailableNodeSelectRule.java | 68 -- .../rule/ConcurrencyNodeSelectRule.java | 37 - .../am/selector/rule/NewECMStandbyRule.java | 82 -- .../selector/rule/ResourceNodeSelectRule.java | 124 --- .../am/selector/rule/ScoreNodeSelectRule.java | 70 -- .../selector/rule/TaskInfoNodeSelectRule.java | 73 -- .../am/service/ECResourceInfoService.java | 17 +- .../ConfCacheRemoveBroadcastListener.java | 75 -- .../service/em/DefaultECMOperateService.java | 37 - .../am/service/em/DefaultEMEngineService.java | 231 ----- .../am/service/em/DefaultEMInfoService.java | 142 --- .../service/em/DefaultEMRegisterService.java | 124 --- .../em/DefaultEMUnregisterService.java | 79 -- .../am/service/em/ECMOperateService.java | 27 - .../am/service/em/EMRegisterService.java | 26 - .../am/service/em/EMUnregisterService.java | 28 - .../service/engine/AbstractEngineService.java | 41 - .../engine/DefaultEngineAskEngineService.java | 198 ---- .../DefaultEngineConnCanKillService.java | 196 ---- .../engine/DefaultEngineCreateService.java | 441 -------- .../engine/DefaultEngineInfoService.java | 145 --- .../engine/DefaultEngineOperateService.java | 39 - .../engine/DefaultEngineRecycleService.java | 92 -- .../engine/DefaultEngineReuseService.java | 272 ----- .../engine/DefaultEngineStopService.java | 316 ------ .../EngineConnStatusCallbackService.java | 25 - .../am/service/engine/EngineInfoService.java | 52 - .../service/engine/EngineOperateService.java | 28 - .../am/service/engine/EngineStopService.java | 61 -- .../service/heartbeat/AMHeartbeatService.java | 86 -- ...efaultEngineConnStatusCallbackService.java | 39 +- .../impl/ECResourceInfoServiceImpl.java | 86 +- .../service/monitor/NodeHeartbeatMonitor.java | 372 ------- .../linkis/manager/am/util/EMUtils.java | 123 +++ .../linkis/manager/am/util/LinkisUtils.java | 248 ----- .../linkis/manager/am/utils/AMUtils.java | 347 ------- .../manager/am/utils/DefaultRetryHandler.java | 122 --- .../linkis/manager/am/vo/AMEngineNodeVo.java | 7 +- .../linkis/manager/am/vo/CanCreateECRes.java | 99 ++ .../apache/linkis/manager/am/vo/ConfigVo.java | 63 +- .../apache/linkis/manager/am/vo/EMNodeVo.java | 16 +- .../manager/label/conf/LabelManagerConf.java | 29 - .../label/score/DefaultNodeLabelScorer.java | 205 ---- .../label/service/NodeLabelRemoveService.java | 25 - .../label/service/NodeLabelService.java | 86 -- .../impl/DefaultNodeLabelAddService.java | 69 -- .../impl/DefaultNodeLabelRemoveService.java | 82 -- .../service/impl/DefaultNodeLabelService.java | 497 --------- .../impl/DefaultResourceLabelService.java | 161 --- .../service/impl/DefaultUserLabelService.java | 167 --- .../manager/rm/conf/ResourceStatus.java | 4 +- .../manager/rm/domain/RMLabelContainer.java | 52 +- .../rm/entity/LabelResourceMapping.java | 68 -- .../manager/rm/exception/RMErrorCode.java | 30 +- .../KubernetesResourceRequester.java | 209 ---- .../KubernetesResourceIdentifierParser.java | 38 - .../impl/ExternalResourceServiceImpl.java | 20 +- .../rm/external/yarn/YarnQueueInfo.java | 86 ++ .../external/yarn/YarnResourceRequester.java | 95 +- .../manager/rm/message/RMMessageService.java | 73 -- .../manager/rm/restful/RMMonitorRest.java | 880 ---------------- .../rm/service/RequestResourceService.java | 473 --------- .../rm/service/ResourceLockService.java | 95 -- .../manager/rm/service/impl/ChangeType.java | 40 - .../impl/DefaultReqResourceService.java | 37 - .../service/impl/DefaultResourceManager.java | 886 ---------------- ...DriverAndKubernetesReqResourceService.java | 133 --- .../impl/DriverAndYarnReqResourceService.java | 151 --- .../impl/LabelResourceServiceImpl.java | 102 -- .../rm/service/impl/ResourceLogService.java | 217 ---- .../rm/service/impl/UserResourceService.java | 128 --- .../linkis/manager/rm/utils/RMUtils.java | 288 ------ .../rm/utils/RequestKerberosUrlUtils.java | 6 + .../manager/rm/utils/UserConfiguration.java | 227 ----- .../EngineConnBmlResourceMapper.xml | 44 +- .../ExternalResourceProviderDaoImpl.xml | 4 +- .../mysql/EngineConnBmlResourceMapper.xml | 93 -- .../loader/EngineConnPluginLoaderConf.scala | 44 + .../conf/EngineConnPluginConfiguration.scala | 42 + .../EngineConnLaunchInterceptor.scala} | 14 +- .../loader/EngineConnPluginsLoader.scala | 32 + ...stractEngineConnBmlResourceGenerator.scala | 123 +++ ...efaultEngineConnBmlResourceGenerator.scala | 109 ++ .../EngineConnBmlResourceGenerator.scala} | 27 +- .../DefaultEngineConnLaunchService.scala | 90 ++ ...aultEngineConnResourceFactoryService.scala | 64 ++ .../DefaultEngineConnResourceService.scala | 324 ++++++ .../service/EngineConnLaunchService.scala} | 15 +- .../EngineConnResourceFactoryService.scala} | 19 +- .../service/EngineConnResourceService.scala | 78 ++ .../manager/am/hook/AskEngineConnHook.scala | 9 +- .../am/hook/AskEngineConnHookContext.scala} | 12 +- .../am/selector/DefaultNodeSelector.scala | 75 ++ .../manager/am/selector/ECAvailableRule.scala | 55 + .../manager/am/selector/NodeSelector.scala} | 15 +- .../rule/AvailableNodeSelectRule.scala | 60 ++ .../rule/ConcurrencyNodeSelectRule.scala | 34 +- .../selector/rule/HotspotExclusionRule.scala} | 56 +- .../am/selector/rule/NewECMStandbyRule.scala | 83 ++ .../am/selector/rule/NodeSelectRule.scala} | 9 +- .../rule/OverLoadNodeSelectRule.scala | 66 ++ .../rule/ResourceNodeSelectRule.scala | 101 ++ .../selector/rule/ScoreNodeSelectRule.scala | 55 + .../rule/TaskInfoNodeSelectRule.scala | 63 ++ .../manager/am/service/EMEngineService.scala} | 27 +- .../manager/am/service/EngineService.scala | 12 +- .../am/service/HeartbeatService.scala} | 9 +- .../ConfCacheRemoveBroadcastListener.scala | 70 ++ .../service/em/DefaultECMOperateService.scala | 36 + .../service/em/DefaultEMEngineService.scala | 216 ++++ .../am/service/em/DefaultEMInfoService.scala | 310 ++++++ .../service/em/DefaultEMRegisterService.scala | 121 +++ .../em/DefaultEMUnregisterService.scala | 84 ++ .../am/service/em/ECMOperateService.scala} | 13 +- .../am/service/em/EMInfoService.scala} | 24 +- .../am/service/em/EMRegisterService.scala} | 23 +- .../am/service/em/EMUnregisterService.scala | 12 +- .../engine/AbstractEngineService.scala | 30 +- .../DefaultEngineAskEngineService.scala | 329 ++++++ .../DefaultEngineConnCanKillService.scala | 173 ++++ .../engine/DefaultEngineCreateService.scala | 474 +++++++++ .../engine/DefaultEngineInfoService.scala | 161 +++ .../engine/DefaultEngineOperateService.scala | 45 +- .../engine/DefaultEngineRecycleService.scala | 82 ++ .../engine/DefaultEngineReuseService.scala | 270 +++++ .../engine/DefaultEngineStopService.scala | 343 +++++++ .../engine/DefaultEngineSwitchService.scala} | 18 +- .../engine/EngineAskEngineService.scala | 33 +- .../engine/EngineConnCanKillService.scala} | 9 +- .../service/engine/EngineCreateService.scala} | 17 +- .../am/service/engine/EngineInfoService.scala | 53 + .../am/service/engine/EngineKillService.scala | 22 +- .../service/engine/EngineOperateService.scala | 24 +- .../engine/EngineRecycleService.scala} | 15 +- .../service/engine/EngineReuseService.scala} | 17 +- .../am/service/engine/EngineStopService.scala | 75 ++ .../service/engine/EngineSwitchService.scala} | 14 +- .../heartbeat/AMHeartbeatService.scala | 112 +++ .../monitor/NodeHeartbeatMonitor.scala | 493 +++++++++ .../linkis/manager/am/utils/AMUtils.scala | 310 ++++++ .../manager/label/LabelManagerUtils.scala} | 26 +- .../label/score/DefaultNodeLabelScorer.scala | 182 ++++ .../score/LabelScoreServiceInstance.scala | 35 + .../label/score/NodeLabelScorer.scala} | 35 +- .../label/service/NodeLabelAddService.scala} | 12 +- .../service/NodeLabelRemoveService.scala | 26 + .../label/service/NodeLabelService.scala | 103 ++ .../label/service/ResourceLabelService.scala} | 32 +- .../label/service/UserLabelService.scala | 30 +- .../impl/DefaultNodeLabelAddService.scala | 56 ++ .../impl/DefaultNodeLabelRemoveService.scala | 75 ++ .../impl/DefaultNodeLabelService.scala | 549 ++++++++++ .../impl/DefaultResourceLabelService.scala | 161 +++ .../impl/DefaultUserLabelService.scala | 123 +++ .../manager/rm/entity/LabelResourceMap.scala} | 39 +- .../rm/entity/ResourceOperationType.scala} | 10 +- .../manager/rm/message/RMMessageService.scala | 68 ++ .../manager/rm/restful/RMMonitorRest.scala | 843 ++++++++++++++++ .../rm/service/LabelResourceService.scala} | 39 +- .../rm/service/RequestResourceService.scala | 531 ++++++++++ .../rm/service/ResourceLockService.scala | 84 ++ .../manager/rm/service/ResourceManager.scala} | 68 +- .../impl/DefaultReqResourceService.scala | 21 +- .../service/impl/DefaultResourceManager.scala | 950 ++++++++++++++++++ .../DriverAndYarnReqResourceService.scala | 187 ++++ .../impl/LabelResourceServiceImpl.scala | 106 ++ .../rm/service/impl/ResourceLogService.scala | 234 +++++ .../rm/service/impl/UserResourceService.scala | 116 +++ .../utils/AcrossClusterRulesJudgeUtils.scala | 246 +++++ .../linkis/manager/rm/utils/RMUtils.scala | 365 +++++++ .../manager/rm/utils/UserConfiguration.scala | 174 ++++ .../service/common/label/LabelChecker.scala} | 16 +- .../service/common/label/LabelFilter.scala} | 13 +- .../common/label/ManagerLabelService.scala} | 17 +- .../src/test/java/org/apache/linkis/Scan.java | 26 - .../apache/linkis/WebApplicationServer.java | 34 - .../engineplugin/server/dao/BaseDaoTest.java | 31 - .../dao/EngineConnBmlResourceDaoTest.java | 111 -- ...Test.java => ECResourceInfoUtilsTest.java} | 2 +- .../dao/ExternalResourceProviderDaoTest.java | 40 - .../src/test/resources/application.properties | 39 - .../src/test/resources/create.sql | 47 - .../src/test/resources/create_pg.sql | 48 - .../src/test/resources/linkis.properties | 25 - .../label/conf/LabelManagerConfTest.scala | 35 + .../linkis/manager/rm/utils/RMUtilsTest.scala | 43 + .../manager/label/conf/LabelCommonConfig.java | 6 +- .../manager/label/entity/engine/RunType.scala | 2 +- .../manager/label/utils/LabelUtil.scala | 21 + .../manager/label/TestLabelBuilder.java | 5 +- .../entity/engine/EngineTypeLabelTest.java | 65 -- .../common/conf/ManagerCommonConf.java | 35 - .../manager/common/conf/RMConfiguration.java | 49 - .../manager/common/constant/AMConstant.java | 6 + .../common/entity/metrics/AMNodeMetrics.java | 11 + .../common/entity/metrics/NodeMetrics.java | 2 + .../manager/common/entity/node/AMEMNode.java | 11 + .../common/entity/node/AMEngineNode.java | 12 + .../manager/common/entity/node/AMNode.java | 4 + .../entity/persistence/PersistenceLabel.java | 17 +- .../persistence/PersistenceNodeMetrics.java | 11 + .../persistence/PersistencerEcNodeInfo.java | 4 +- .../common/entity/resource/CPUResource.java | 2 +- .../entity/resource/CommonNodeResource.java | 30 + .../resource/DriverAndYarnResource.java | 111 +- .../common/entity/resource/LoadResource.java | 2 +- .../entity/resource/MemoryResource.java | 2 +- .../common/entity/resource/NodeResource.java | 12 + .../common/entity/resource/Resource.java | 16 - .../common/entity/resource/ResourceType.java | 2 + .../entity/resource/SpecialResource.java | 4 +- .../ManagerCommonErrorCodeSummary.java | 3 +- .../manager/common/operator/Operator.java | 56 -- .../common/operator/OperatorFactoryImpl.java | 72 -- .../common/protocol/RequestManagerUnlock.java | 59 -- .../common/protocol/ResponseEngineLock.java | 56 -- .../common/protocol/ResponseEngineUnlock.java | 36 - .../common/protocol/em/ECMOperateRequest.java | 51 - .../protocol/em/ECMOperateResponse.java | 52 - .../em/EMResourceRegisterRequest.java | 79 -- .../common/protocol/em/RegisterEMRequest.java | 79 -- .../protocol/em/RegisterEMResponse.java | 50 - .../engine/EngineAskAsyncResponse.java | 59 -- .../protocol/engine/EngineAsyncResponse.java | 24 - .../engine/EngineConnStatusCallback.java | 68 -- .../engine/EngineConnStatusCallbackToAM.java | 69 -- .../protocol/engine/EngineCreateError.java | 66 -- .../protocol/engine/EngineCreateSuccess.java | 53 - .../protocol/engine/EngineOperateRequest.java | 42 - .../engine/EngineOperateResponse.java | 53 - .../protocol/label/LabelReportRequest.java | 43 - .../protocol/node/NodeHeartbeatMsg.java | 7 +- .../protocol/resource/ResourceProtocol.java | 22 - .../resource/ResourceUsedProtocol.java | 51 - .../protocol/resource/ResourceWithStatus.java | 103 -- .../resource/ResponseTaskRunningInfo.java | 74 -- .../manager/common/utils/ManagerUtils.java | 7 - .../manager/common/utils/ResourceUtils.java | 30 +- .../linkis/manager/rm/NotEnoughResource.java | 36 - .../manager/rm/RequestResourceAndWait.java | 47 - .../manager/common/operator/Operator.scala | 44 + .../common/operator/OperatorFactory.scala | 64 ++ .../manager/common/protocol/EngineLock.scala} | 39 +- .../common/protocol/OperateRequest.scala} | 34 +- .../common/protocol/OperateResponse.scala | 13 +- .../protocol/em/ECMOperateRequest.scala} | 32 +- .../protocol/em/ECMOperateResponse.scala | 32 + .../em/EMResourceRegisterRequest.scala | 67 ++ .../protocol/em/RegisterEMRequest.scala | 67 ++ .../protocol/em/RegisterEMResponse.scala | 24 + .../protocol/engine/EngineAsyncResponse.scala | 38 + .../engine/EngineConnStatusCallback.scala | 55 + .../engine/EngineOperateRequest.scala | 31 + .../engine/EngineOperateResponse.scala | 32 + .../protocol/label/LabelUpdateRequest.scala | 21 +- .../protocol/resource/ResourceProtocol.scala | 57 ++ .../resource/ResourceWithStatus.scala | 21 +- .../linkis/manager/rm/ResourceInfo.scala} | 8 +- .../linkis/manager/rm/ResultResource.scala | 9 +- .../manager/dao/ECResourceRecordMapper.java | 3 +- .../linkis/manager/dao/LockManagerMapper.java | 7 + .../linkis/manager/dao/NodeManagerMapper.java | 4 +- ...kisManagerPersistenceErrorCodeSummary.java | 4 +- .../persistence/NodeManagerPersistence.java | 21 +- .../NodeMetricManagerPersistence.java | 12 +- .../ResourceManagerPersistence.java | 6 +- .../impl/DefaultLabelManagerPersistence.java | 8 + .../impl/DefaultLockManagerPersistence.java | 30 +- .../impl/DefaultNodeManagerPersistence.java | 70 +- .../DefaultNodeMetricManagerPersistence.java | 27 +- .../impl/DefaultResourceLabelPersistence.java | 17 +- .../DefaultResourceManagerPersistence.java | 12 +- .../manager/util/PersistenceManagerConf.java | 5 - .../mapper/common/ECResourceRecordMapper.xml | 109 ++ .../LabelManagerMapper.xml | 54 +- .../mapper/common/NodeManagerMapper.xml | 49 +- .../mapper/common/NodeMetricManagerMapper.xml | 9 +- .../ResourceManagerMapper.xml | 23 +- .../mapper/mysql/ECResourceRecordMapper.xml | 129 --- .../mapper/mysql/LabelManagerMapper.xml | 585 ----------- .../mapper/mysql/ResourceManagerMapper.xml | 148 --- .../postgresql/ECResourceRecordMapper.xml | 127 --- .../dao/ECResourceRecordMapperTest.java | 134 --- .../manager/dao/LabelManagerMapperTest.java | 293 ------ .../manager/dao/NodeManagerMapperTest.java | 1 - .../dao/ResourceManagerMapperTest.java | 45 +- .../src/test/resources/application.properties | 20 +- .../src/test/resources/create.sql | 55 +- .../src/test/resources/create_pg.sql | 189 ---- .../executor/DorisEngineConnExecutor.java | 4 +- ...cSearchProcessEngineConnLaunchBuilder.java | 2 +- .../ElasticSearchEngineConnExecutor.java | 4 +- .../executor/client/ResponseHandler.scala | 11 +- .../client/impl/ResponseHandlerImpl.scala | 6 +- .../errorcode/FlinkErrorCodeSummary.java | 132 +++ .../exception/ExecutorInitException.java | 45 + .../exception/FlinkInitFailedException.java | 36 + .../exception/JobExecutionException.java | 36 + .../exception/SqlExecutionException.java | 36 + .../flink/exception/SqlParseException.java} | 18 +- .../flink/executor/FlinkExecutor.scala | 2 +- .../FlinkManagerConcurrentExecutor.scala | 103 ++ .../flink/operator/KillOperator.scala | 32 +- .../flink/operator/ListOperator.scala | 17 +- .../flink/operator/StatusOperator.scala | 22 +- .../operator/TriggerSavepointOperator.scala | 25 +- .../FlinkRestClientManager.scala | 21 +- .../hive/src/main/resources/log4j2.xml | 4 +- .../hive/creation/HiveEngineConnFactory.scala | 2 +- .../HiveEngineConcurrentConnExecutor.scala | 4 +- .../executor/HiveEngineConnExecutor.scala | 11 +- .../impala/conf/ImpalaConfiguration.scala | 2 +- .../io_file/src/main/resources/log4j2.xml | 6 +- .../io/executor/IoEngineConnExecutor.scala | 97 +- .../io/service/FsProxyService.scala | 18 +- .../engineplugin/io/utils/IOHelp.scala | 34 +- .../executor/IoEngineConnExecutorTest.java | 77 -- .../src/test/resources/testIoResult.dolphin | 0 .../engineplugin/jdbc/ConnectionManager.java | 6 - .../jdbc/errorcode/JDBCErrorCodeSummary.java | 4 +- .../JDBCProcessEngineConnLaunchBuilder.scala | 2 +- .../jdbc/conf/JDBCConfiguration.scala | 6 + .../executor/JDBCEngineConnExecutor.scala | 24 +- .../jdbc/executor/JDBCHelper.java | 38 +- .../executor/JDBCMultiDatasourceParser.scala | 22 +- .../nebula/src/main/assembly/distribution.xml | 2 +- .../executor/NebulaEngineConnExecutor.java | 6 +- .../OpenLooKengProcessECLaunchBuilder.java | 2 +- .../OpenLooKengEngineConnExecutor.java | 2 +- .../pipeline/src/main/resources/log4j2.xml | 3 + .../executor/PipelineEngineConnExecutor.scala | 3 - .../PrestoProcessEngineConnLaunchBuilder.java | 2 +- .../executor/PrestoEngineConnExecutor.java | 2 +- .../python/PythonEngineConnPlugin.java | 80 -- .../conf/PythonEngineConfiguration.java | 51 - .../LinkisPythonErrorCodeSummary.java | 2 +- .../executor/PythonEngineConnExecutor.java | 189 ---- .../python/hook/PythonVersionEngineHook.java | 78 -- .../engineplugin/python/utils/Kind.java | 92 -- .../python/src/main/resources/log4j2.xml | 3 + .../src/main/resources/python/python.py | 4 +- .../python/PythonEngineConnPlugin.scala | 75 ++ .../conf/PythonEngineConfiguration.scala | 49 + .../exception/NoSupportEngineException.scala} | 15 +- .../PythonSessionNullException.scala | 35 + .../executor/PythonEngineConnExecutor.scala | 172 ++++ .../python/executor/PythonSession.scala | 43 +- .../factory/PythonEngineConnFactory.scala | 2 +- .../python/hook/PythonVersionEngineHook.scala | 67 ++ ...ythonProcessEngineConnLaunchBuilder.scala} | 6 +- .../engineplugin/python/utils/Kind.scala | 104 ++ .../engineplugin/python/utils/State.scala | 74 ++ .../TestNoSupportEngineException.java | 47 - .../TestPythonEngineConnExecutor.java | 82 -- .../factory/TestPythonEngineConnFactory.java | 44 - .../python/TestPythonEngineConnPlugin.scala} | 14 +- .../conf/TestPythonEngineConfiguration.scala} | 32 +- .../TestNoSupportEngineException.scala | 95 ++ .../TestPythonEngineConnExecutor.scala | 82 ++ .../factory/TestPythonEngineConnFactory.scala | 42 + .../engineplugin/python/utils/TestKind.scala} | 27 +- .../engineplugin/python/utils/TestState.scala | 32 +- .../repl/executor/ReplEngineConnExecutor.java | 4 +- .../shell/ShellEngineConnPlugin.java | 81 -- .../common/ShellEngineConnPluginConst.java | 24 - .../shell/conf/ShellEngineConnConf.java | 30 - .../exception/ShellCodeErrorException.java | 28 - .../shell/executor/ReaderThread.java | 97 -- .../shell/executor/ShellECTaskInfo.java | 54 - .../ShellEngineConnConcurrentExecutor.java | 151 --- .../executor/ShellEngineConnExecutor.java | 339 ------- .../shell/executor/YarnAppIdExtractor.java | 53 - .../conf/linkis-engineconn.properties | 3 +- .../shell/src/main/resources/conf/log4j2.xml | 5 +- .../shell/ShellEngineConnPlugin.scala | 76 ++ ...ShellProcessEngineConnLaunchBuilder.scala} | 6 +- .../shell/common/ShellEnginePluginConst.scala | 11 +- .../shell/conf/ShellEngineConnConf.scala | 14 +- .../exception/NoCorrectUserException.scala | 27 + .../shell/executor/ReaderThread.scala | 105 ++ .../shell/executor/ShellECTaskInfo.scala | 20 + .../ShellEngineConnConcurrentExecutor.scala | 378 +++++++ .../executor/ShellEngineConnExecutor.scala | 337 +++++++ .../shell/executor/YarnAppIdExtractor.scala | 81 ++ .../factory/ShellEngineConnFactory.scala | 17 +- .../executor/TestShellEngineConnExecutor.java | 62 -- .../shell/TestShellEngineConnPlugin.scala} | 20 +- .../TestShellEngineConnPluginConst.scala} | 17 +- .../TestNoCorrectUserException.scala} | 20 +- .../TestShellEngineConnExecutor.scala | 62 ++ .../spark/datacalc/util/PluginUtil.java | 1 - .../resources/linkis-engineconn.properties | 2 +- .../src/main/resources/python/mix_pyspark.py | 31 +- .../engineplugin/spark/common/SparkKind.scala | 22 - .../spark/config/SparkConfiguration.scala | 5 +- .../spark/datacalc/source/SolrSource.scala | 44 - .../spark/executor/SQLSession.scala | 4 +- .../executor/SparkEngineConnExecutor.scala | 42 +- .../spark/executor/SparkPythonExecutor.scala | 8 +- .../spark/executor/SparkScalaExecutor.scala | 4 +- .../executor/SparkSubmitOnceExecutor.scala | 2 +- .../factory/SparkEngineConnFactory.scala | 4 + .../spark/imexport/ExportData.scala | 13 +- .../spark/imexport/LoadData.scala | 57 +- ...SubmitProcessEngineConnLaunchBuilder.scala | 18 +- .../spark/mdq/MDQPostExecutionHook.scala | 14 +- .../spark/mdq/MDQPreExecutionHook.scala | 17 +- .../spark/utils/EngineUtils.scala | 4 +- .../datasources/csv/DolphinToSpark.scala | 26 +- .../spark/src/test/resources/etltest.dolphin | 4 - .../spark/datacalc/TestExcelCala.scala | 135 --- .../spark/datacalc/TestKafkaCala.scala | 118 --- .../spark/datacalc/TestRedisCalc.scala | 200 ---- .../spark/datacalc/TestRocketmqCala.scala | 71 -- .../sqoop/src/main/assembly/distribution.xml | 4 - .../sqoop/client/config/ParamsMapping.java | 6 + .../sqoop/src/main/resources/log4j2.xml | 18 +- .../sqoop/client/RemoteClientHolder.scala | 97 ++ .../DataSourceRpcErrorException.scala | 21 +- .../executor/SqoopOnceCodeExecutor.scala | 10 +- .../sqoop/params/ConnectParamsResolver.scala | 82 ++ .../SqoopDataSourceParamsResolver.scala | 143 +++ .../trino/TrinoEngineConnPlugin.java | 78 -- .../trino/conf/TrinoConfiguration.java | 90 -- .../trino/conf/TrinoEngineConfig.java | 53 - .../trino/exception/TrinoClientException.java | 26 - .../exception/TrinoGrantmaException.java | 26 - .../exception/TrinoModifySchemaException.java | 26 - .../exception/TrinoStateInvalidException.java | 26 - .../executor/TrinoEngineConnExecutor.java | 561 ----------- .../password/CommandPasswordCallback.java | 4 +- .../password/StaticPasswordCallback.java | 2 +- .../engineplugin/trino/utils/TrinoCode.java | 47 - .../resources/linkis-engineconn.properties | 1 - .../trino/src/main/resources/log4j2.xml | 3 + .../trino/TrinoEngineConnPlugin.scala | 66 ++ ...TrinoProcessEngineConnLaunchBuilder.scala} | 24 +- .../trino/conf/TrinoConfiguration.scala | 73 ++ .../trino/conf/TrinoEngineConfig.scala | 48 + .../trino/exception/TrinoException.scala | 32 + .../executor/TrinoEngineConnExecutor.scala | 563 +++++++++++ .../engineplugin/trino/utils/TrinoCode.scala | 65 ++ .../trino/utils/TrinoSQLHook.scala} | 18 +- .../src/main/assembly/distribution.xml | 3 - .../service/impl/CleanerServiceImpl.java | 3 +- .../config/ApplicationConfiguration.java | 48 + .../linkis/monitor/config/MonitorConfig.java | 5 +- .../monitor/jobhistory/entity/JobHistory.java | 186 +++- .../linkis/monitor/scheduled/BmlClear.java | 17 + .../scheduled/EntranceTaskMonitor.java | 47 +- .../monitor/scheduled/JobHistoryClear.java | 1 + .../monitor/scheduled/JobHistoryMonitor.java | 127 +-- .../monitor/scheduled/ResourceClear.java | 49 + .../monitor/scheduled/ResourceMonitor.java | 199 ++-- .../monitor/scheduled/TaskArchiveClear.java | 53 + .../linkis/monitor/until/JobMonitorUtils.java | 88 ++ .../mapper/common/JobHistoryMapper.xml | 60 +- ...kisJobHistoryScanSpringConfiguration.scala | 5 +- .../linkis/monitor/constants/Constants.scala | 7 +- .../jobhistory/JobHistoryDataFetcher.scala | 5 +- .../errorcode/JobHistoryErrCodeRule.scala | 25 +- .../jobhistory/index/JobIndexHitEvent.scala | 22 + .../jobhistory/index/JobIndexRule.scala | 82 ++ .../jobhistory/index/JobIndexSender.scala | 25 + .../jobtime/JobTimeExceedAlertSender.scala | 3 +- .../labels/JobHistoryLabelsAlertSender.scala | 6 +- .../labels/JobHistoryLabelsRule.scala | 76 +- .../runtime/CommonJobRunTimeRule.scala | 15 +- .../runtime/JobHistoryRunTimeRule.scala | 15 +- .../utils/alert/ims/MonitorAlertUtils.scala | 27 +- .../utils/alert/ims/UserLabelAlertUtils.scala | 47 - .../storage/io/client/DefaultIOClient.scala | 4 +- .../io/iteraceptor/IOMethodInterceptor.scala | 14 +- .../IOMethodInterceptorCreatorImpl.scala | 4 +- .../storage/io/utils/IOClientUtils.scala | 3 +- .../DefaultCodeExecTaskExecutorManager.scala | 7 +- .../monitor/EngineConnMonitor.scala | 3 +- .../physical/CodeLogicalUnitExecTask.scala | 6 +- .../ComputationTaskExecutionReceiver.scala | 10 +- .../ecm/ComputationEngineConnManager.scala | 54 +- .../ecm/cache/EngineAsyncResponseCache.scala | 16 +- .../orchestrator/ecm/conf/ECMPluginConf.scala | 2 + .../impl/ComputationEngineConnExecutor.scala | 2 +- .../DefaultEngineAsyncResponseService.scala | 14 +- .../execution/AbstractExecution.scala | 26 - .../execution/impl/DefaultTaskManager.scala | 47 +- .../execution/impl/ExecutionImpl.scala | 27 +- .../execution/ExecutionTaskEvent.scala | 13 +- .../bml/service/impl/TaskServiceImpl.java | 11 +- .../BmlProjectMapper.xml | 73 +- .../mapper/common/DownloadMapper.xml | 6 +- .../{mysql => common}/ResourceMapper.xml | 32 +- .../{postgresql => common}/TaskMapper.xml | 8 +- .../{postgresql => common}/VersionMapper.xml | 85 +- .../mapper/mysql/BmlProjectMapper.xml | 78 -- .../resources/mapper/mysql/TaskMapper.xml | 69 -- .../resources/mapper/mysql/VersionMapper.xml | 243 ----- .../mapper/postgresql/ResourceMapper.xml | 110 -- .../linkis/bml/dao/BmlProjectDaoTest.java | 5 - .../src/test/resources/application.properties | 20 +- .../src/test/resources/create_pg.sql | 129 --- .../dao/AcrossClusterRuleMapper.java | 3 +- .../api/AcrossClusterRuleRestfulApi.java | 5 +- .../restful/api/ConfigurationRestfulApi.java | 14 +- .../restful/api/TemplateRestfulApi.java | 3 + .../api/TenantConfigrationRestfulApi.java | 9 +- .../api/UserIpConfigrationRestfulApi.java | 2 +- .../service/AcrossClusterRuleService.java | 7 +- .../service/TenantConfigService.java | 2 + .../impl/AcrossClusterRuleServiceImpl.java | 10 +- .../impl/TemplateConfigKeyServiceImpl.java | 5 +- .../service/impl/TenantConfigServiceImpl.java | 8 +- .../service/impl/UserIpConfigServiceImpl.java | 2 +- .../AcrossClusterRuleMapper.xml | 2 + .../ConfigKeyLimitForUserMapper.xml | 0 .../mapper/{mysql => common}/ConfigMapper.xml | 4 +- .../DepartmentTenantMapper.xml} | 68 +- .../resources/mapper/common/LabelMapper.xml | 18 +- .../TemplateConfigKeyMapper.xml | 0 .../mapper/{mysql => common}/UserIpMapper.xml | 12 +- .../{mysql => common}/UserTenantMapper.xml | 4 +- .../postgresql/AcrossClusterRuleMapper.xml | 130 --- .../ConfigKeyLimitForUserMapper.xml | 145 --- .../mapper/postgresql/ConfigMapper.xml | 485 --------- .../postgresql/TemplateConfigKeyMapper.xml | 161 --- .../mapper/postgresql/UserIpMapper.xml | 91 -- .../configuration/conf/Configuration.scala | 8 - .../configuration/constant/Constants.scala | 5 +- .../service/CategoryService.scala | 7 +- .../configuration/dao/ConfigMapperTest.java | 17 + .../configuration/dao/LabelMapperTest.java | 17 + .../exception/ConfigurationExceptionTest.java | 17 + .../api/ConfigurationRestfulApiTest.java | 8 +- .../src/test/resources/application.properties | 20 +- .../src/test/resources/create.sql | 2 +- .../src/test/resources/create_pg.sql | 148 --- .../ContextValueTypeConditionParser.java | 19 +- .../linkis/cs/conf/CSConfiguration.java | 12 +- .../contextHistoryMapper.xml | 22 +- .../contextIDListenerMapper.xml | 11 +- .../{mysql => common}/contextIDMapper.xml | 4 +- .../common/contextKeyListenerMapper.xml | 4 +- .../{mysql => common}/contextMapMapper.xml | 6 +- .../postgresql/contextHistoryMapper.xml | 83 -- .../postgresql/contextIDListenerMapper.xml | 50 - .../mapper/postgresql/contextIDMapper.xml | 110 -- .../mapper/postgresql/contextMapMapper.xml | 172 ---- .../dao/ContextHistoryMapperTest.java | 6 +- .../persistence/dao/ContextIDMapperTest.java | 12 +- .../persistence/dao/ContextMapMapperTest.java | 266 ----- .../cs/server/conf/ContextServerConfTest.java | 2 + .../src/test/resources/application.properties | 19 +- .../src/test/resources/application.yml | 13 - .../src/test/resources/create.sql | 18 +- .../src/test/resources/create_pg.sql | 90 -- .../label/InsLabelAutoConfiguration.java | 6 +- .../label/restful/InstanceRestful.java | 23 +- .../service/impl/SpringInsLabelService.java | 2 - .../InsLabelRelationMapper.xml | 77 +- .../{mysql => common}/InstanceInfoMapper.xml | 22 +- .../{mysql => common}/InstanceLabelMapper.xml | 40 +- .../postgresql/InsLabelRelationMapper.xml | 205 ---- .../mapper/postgresql/InstanceInfoMapper.xml | 55 - .../mapper/postgresql/InstanceLabelMapper.xml | 114 --- .../label/dao/InsLabelRelationDaoTest.java | 10 - .../label/dao/InstanceInfoDaoTest.java | 5 - .../label/dao/InstanceLabelDaoTest.java | 5 - .../src/test/resources/application.properties | 5 +- .../src/test/resources/create_pg.sql | 65 -- .../jobhistory/dao/JobHistoryMapper.java | 29 +- .../linkis/jobhistory/entity/JobHistory.java | 159 +-- .../linkis/jobhistory/entity/QueryTaskVO.java | 46 + .../restful/api/QueryRestfulApi.java | 427 ++++++-- .../jobhistory/util/JobhistoryUtils.java | 169 ++++ .../mapper/common/JobDetailMapper.xml | 22 +- .../mapper/mysql/JobHistoryMapper.xml | 121 ++- .../conf/JobhistoryConfiguration.scala | 5 + .../conversions/TaskConversions.scala | 63 +- .../service/JobHistoryQueryService.java | 16 +- .../impl/JobHistoryQueryServiceImpl.scala | 78 +- .../linkis/jobhistory/util/QueryUtils.scala | 68 +- .../jobhistory/dao/JobDetailMapperTest.java | 95 -- .../jobhistory/dao/JobHistoryMapperTest.java | 6 +- .../restful/api/QueryRestfulApiTest.java | 179 ---- .../service/JobHistoryQueryServiceTest.java | 12 +- .../src/test/resources/application.properties | 20 +- .../src/test/resources/create.sql | 4 +- .../src/test/resources/create_pg.sql | 62 -- .../errorcode/client/ClientConfiguration.java | 2 +- .../linkis/bml/conf/BmlConfiguration.scala | 2 +- .../cs/client/utils/ContextClientConf.scala | 2 +- .../config/DatasourceClientConfig.scala | 2 +- .../filesystem/conf/WorkspaceClientConf.scala | 5 +- .../BmlClientErrorCodeSummaryTest.java | 45 - .../builder/ContextClientFactoryTest.java | 35 - .../builder/HttpContextClientConfigTest.java | 33 - .../CsClientErrorCodeSummaryTest.java | 59 -- .../apache/linkis/cs/client/test/Test.java | 134 --- .../test/bean/ClientTestContextKey.java | 67 -- .../client/test/bean/ResourceValueBean.java | 45 - .../listener/CommonContextKeyListener.java | 40 - .../test/no_context_search/TestClear.java | 60 -- .../cs/client/test/restful/RestfulTest.java | 146 --- .../cs/client/test/service/TestInfo.java | 99 -- .../cs/client/test/service/TestRemove.java | 113 --- .../test/service/TestSearchService.java | 113 --- .../test_multiuser/TestChangeContext.java | 110 -- .../test_multiuser/TestCreateContext.java | 122 --- .../client/utils/ContextClientConfTest.java | 36 - .../client/utils/ContextServiceUtilsTest.java | 103 -- .../cs/client/utils/SerializeHelperTest.java | 60 -- .../client/TestDataSourceClient.scala | 123 --- .../datasource/client/TestHiveClient.scala | 144 --- .../datasource/client/TestMysqlClient.scala | 149 --- .../config/DatasourceClientConfigTest.java | 49 - .../DataSourceClientBuilderExceptionTest.java | 34 - .../imp/LinkisDataSourceRemoteClientTest.java | 37 - .../client/ClientConfigurationTest.java | 46 - .../client/ErrorCodeClientBuilderTest.java | 42 - .../client/LinkisErrorCodeClientTest.java | 44 - .../client/handler/ErrorCodeHandlerTest.java | 37 - .../handler/LinkisErrorCodeHandlerTest.java | 48 - .../manager/LinkisErrorCodeManagerTest.java | 45 - .../LinkisErrorCodeSynchronizerTest.java | 46 - .../client/action/ErrorCodeActionTest.scala | 41 - .../action/OpenScriptFromBMLActionTest.scala | 35 - .../conf/WorkspaceClientConfTest.scala | 45 - .../linkis/udf/entity/PythonModuleInfoVO.java | 209 ++++ .../api/rpc/RequestPythonModuleProtocol.scala | 28 + .../rpc/ResponsePythonModuleProtocol.scala | 22 +- .../entity/enumeration/ContextScopeTest.java | 40 - .../entity/enumeration/ContextTypeTest.java | 48 - .../common/entity/enumeration/DBTypeTest.java | 40 - .../entity/enumeration/WorkTypeTest.java | 40 - .../protocol/ContextHistoryTypeTest.java | 36 - .../cs/common/protocol/ContextIDTypeTest.java | 43 - .../common/protocol/ContextKeyTypeTest.java | 37 - .../protocol/ContextKeyValueTypeTest.java | 37 - .../common/protocol/ContextValueTypeTest.java | 37 - .../ContextSerializationHelperTest.java | 199 ---- ...CombinedNodeIDContextIDSerializerTest.java | 59 -- .../CommonContextKeySerializerTest.java | 60 -- .../LinkisBMLResourceSerializerTest.java | 63 -- .../data/CSResultDataSerializerTest.java | 60 -- .../data/LinkisJobDataSerializerTest.java | 61 -- .../value/metadata/CSTableSerializerTest.java | 65 -- .../object/CSFlowInfosSerializerTest.java | 57 -- .../test/ContextSerializationHelperTest.java | 187 ---- .../cs/common/utils/CSCommonUtilsTest.java | 67 -- .../utils/CSHighAvailableUtilsTest.java | 64 -- .../cs/listener/test/TestContextID.java | 35 - .../cs/listener/test/TestContextKeyValue.java | 49 - .../errorcode/common/CommonConfTest.java | 33 - .../query/common/MdmConfigurationTest.java | 37 - .../common/cache/CacheConfigurationTest.java | 40 - .../common/cache/ConnCacheManagerTest.java | 48 - .../MetaMethodInvokeExceptionTest.java | 34 - .../exception/MetaRuntimeExceptionTest.java | 34 - .../udf/excepiton/UDFExceptionTest.java | 33 - .../linkis/udf/utils/ConstantVarTest.java | 48 - .../server/response/EngineLabelResponse.java | 2 +- .../restful/GatewayAuthTokenRestfulApi.java | 46 + .../service/GatewayAuthTokenService.java | 2 + .../impl/GatewayAuthTokenServiceImpl.java | 8 + .../conf/WorkSpaceConfiguration.java | 15 +- .../constant/WorkSpaceConstants.java | 8 + .../exception/WorkspaceExceptionManager.java | 18 + .../restful/api/BMLFsRestfulApi.java | 11 +- .../filesystem/restful/api/FsRestfulApi.java | 557 +++++++++- .../linkis/filesystem/util/WorkspaceUtil.java | 4 + .../mapper/common/CgManagerLabelMapper.xml | 6 +- .../ConfigurationConfigKeyMapper.xml | 22 +- .../common/ConfigurationConfigValueMapper.xml | 11 +- .../ConfigurationKeyEngineRelationMapper.xml | 5 +- .../DatasourceAccessMapper.xml | 8 +- .../{mysql => common}/DatasourceEnvMapper.xml | 10 +- .../DatasourceTypeKeyMapper.xml | 10 +- .../DatasourceTypeMapper.xml | 12 +- .../EngineConnPluginBmlResourcesMapper.xml | 6 +- .../GatewayAuthTokenMapper.xml | 10 +- .../PsErrorCodeMapper.xml | 10 +- .../RmExternalResourceProviderMapper.xml | 12 +- .../UdfBaseInfoMapper.xml} | 18 +- .../{mysql => common}/UdfManagerMapper.xml | 6 +- .../{mysql => common}/UdfTreeMapper.xml | 12 +- .../mapper/{mysql => common}/VarMapper.xml | 8 +- .../mapper/mysql/PsErrorCodeMapper.xml | 47 - .../ConfigurationConfigKeyMapper.xml | 68 -- .../postgresql/DatasourceAccessMapper.xml | 49 - .../mapper/postgresql/DatasourceEnvMapper.xml | 51 - .../postgresql/DatasourceTypeKeyMapper.xml | 64 -- .../postgresql/DatasourceTypeMapper.xml | 50 - .../postgresql/GatewayAuthTokenMapper.xml | 51 - .../RmExternalResourceProviderMapper.xml | 47 - .../mapper/postgresql/UdfTreeMapper.xml | 51 - .../resources/mapper/postgresql/VarMapper.xml | 61 -- .../linkis/filesystem/service/FsService.scala | 20 + .../filesystem/validator/PathValidator.scala | 10 +- .../linkis/basedatamanager/server/Scan.java | 26 - .../server/WebApplicationServer.java | 34 - .../server/dao/BaseDaoTest.java | 31 - .../dao/ConfigurationConfigKeyMapperTest.java | 86 -- .../ConfigurationConfigValueMapperTest.java | 65 -- ...figurationKeyEngineRelationMapperTest.java | 53 - .../dao/DatasourceAccessMapperTest.java | 48 - .../server/dao/DatasourceEnvMapperTest.java | 55 - .../server/dao/DatasourceTypeMapperTest.java | 39 - ...ngineConnPluginBmlResourcesMapperTest.java | 56 -- .../dao/GatewayAuthTokenMapperTest.java | 55 - .../server/dao/PsErrorCodeMapperTest.java | 50 - .../RmExternalResourceProviderMapperTest.java | 52 - .../server/dao/UdfManagerMapperTest.java | 47 - .../server/dao/UdfTreeMapperTest.java | 54 - .../ConfigurationTemplateRestfulApiTest.java | 143 --- .../DatasourceAccessRestfulApiTest.java | 149 --- .../restful/DatasourceEnvRestfulApiTest.java | 138 --- .../DatasourceTypeKeyRestfulApiTest.java | 158 --- .../restful/DatasourceTypeRestfulApiTest.java | 159 --- .../restful/ErrorCodeRestfulApiTest.java | 154 --- .../GatewayAuthTokenRestfulApiTest.java | 161 --- .../server/restful/MvcUtils.java | 126 --- ...xternalResourceProviderRestfulApiTest.java | 154 --- .../restful/UdfManagerRestfulApiTest.java | 151 --- .../server/restful/UdfTreeRestfulApiTest.java | 159 --- .../service/DatasourceAccessServiceTest.java | 63 -- .../service/DatasourceEnvServiceTest.java | 65 -- .../service/DatasourceTypeKeyServiceTest.java | 65 -- .../service/DatasourceTypeServiceTest.java | 64 -- .../server/service/ErrorCodeServiceTest.java | 63 -- .../service/GatewayAuthTokenServiceTest.java | 63 -- ...RmExternalResourceProviderServiceTest.java | 60 -- .../server/service/UdfManagerServiceTest.java | 59 -- .../server/service/UdfTreeServiceTest.java | 60 -- .../apache/linkis/errorcode/server/Scan.java | 26 - .../server/WebApplicationServer.java | 34 - .../errorcode/server/dao/BaseDaoTest.java | 31 - .../server/dao/ErrorCodeMapperTest.java | 40 - .../restful/api/FsRestfulApiTest.java | 2 +- .../java/org/apache/linkis/variable/Scan.java | 26 - .../linkis/variable/WebApplicationServer.java | 34 - .../linkis/variable/dao/BaseDaoTest.java | 31 - .../linkis/variable/dao/VarMapperTest.java | 124 --- .../linkis/variable/restful/MvcUtils.java | 115 --- .../restful/api/VariableRestfulApiTest.java | 112 --- .../variable/service/VariableServiceTest.java | 85 -- .../src/test/resources/application.properties | 58 +- .../resources/basedata_manager_create.sql | 451 --------- .../resources/basedata_manager_create_pg.sql | 252 ----- .../src/test/resources/error_code_create.sql | 34 - .../test/resources/error_code_create_pg.sql | 32 - .../src/test/resources/info.text | 19 - .../src/test/resources/variable_create.sql | 61 -- .../src/test/resources/variable_create_pg.sql | 59 -- .../common/lock/dao/CommonLockMapperTest.java | 13 + .../src/test/resources/application.properties | 20 +- .../src/test/resources/create.sql | 6 +- .../src/test/resources/create_pg.sql | 31 - .../src/test/resources/data.sql | 22 + .../apache/linkis/udf/api/UDFRestfulApi.java | 280 ++++++ .../udf/dao/PythonModuleInfoMapper.java} | 34 +- .../org/apache/linkis/udf/dao/UDFDao.java | 4 +- .../linkis/udf/entity/PythonModuleInfo.java | 158 +++ .../udf/service/PythonModuleInfoService.java} | 28 +- .../apache/linkis/udf/service/UDFService.java | 2 +- .../impl/PythonModuleInfoServiceImpl.java | 64 ++ .../udf/service/impl/UDFServiceImpl.java | 4 +- .../mapper/common/PythonModuleInfoMapper.xml | 93 ++ .../mapper/{postgresql => common}/UDFDao.xml | 184 ++-- .../resources/mapper/common/UDFTreeDao.xml | 56 +- .../{postgresql => common}/UDFVersionDao.xml | 30 +- .../main/resources/mapper/mysql/UDFDao.xml | 471 --------- .../resources/mapper/mysql/UDFVersionDao.xml | 130 --- .../linkis/udf/api/rpc/UdfReceiver.scala | 56 +- .../udf/api/rpc/UdfReceiverChooser.scala | 9 +- .../udf/api/PythonModuleRestfulApiTest.java | 132 +++ .../udf/dao/PythonModuleInfoMapperTest.java | 113 +++ .../service/PythonModuleInfoServiceTest.java | 129 +++ .../src/test/resources/application.properties | 22 +- .../src/test/resources/create.sql | 57 +- .../src/test/resources/create_pg.sql | 143 --- .../src/test/resources/data.sql | 61 ++ .../conf/TokenConfiguration.scala | 4 +- .../service/CachedTokenService.scala | 2 +- .../gateway/config/GatewayConfiguration.scala | 5 + .../linkis/gateway/security/UserRestful.scala | 12 + .../dws/config/DWSClientConfig.scala | 9 +- .../ujes/parser/ECMRequestGatewayParser.scala | 74 ++ .../route/AbstractLabelGatewayRouter.scala | 31 +- .../http/IpPriorityLoadBalancer.java | 19 +- 1081 files changed, 29102 insertions(+), 43957 deletions(-) delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/FSFactory.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntity.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntitySerializer.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelXlsReader.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/FirstRowDeal.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/RowToCsvDeal.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageExcelWriter.java rename linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/ExecuteException.java => linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColLengthExceedException.java (65%) create mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColumnIndexExceedException.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildOSSSystem.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildS3FileSystem.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorFactory.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetFactory.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/html/HtmlResultSet.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultDeserializer.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSerializer.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSet.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/picture/PictureResultSet.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSet.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSerializer.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSet.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Compaction.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ParserFactory.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsWriter.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Variable.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/VariableParser.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/PYScriptCompaction.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/QLScriptCompaction.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/CommonScriptParser.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/PYScriptParser.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/QLScriptParser.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ScalaScriptParser.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ShellScriptParser.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/reader/StorageScriptFsReader.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/AbstractFileSource.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSplit.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/TextFileSource.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/OrcUtils.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageConfiguration.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java delete mode 100644 linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala rename linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientFactoryTest.scala => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineMetaData.scala (69%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/OperatorFactory.java => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineRecord.scala (67%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/entity/ResourceOperationType.java => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsReader.scala (85%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/AlertUtils.java => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVReader.scala (91%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsReader.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/script/ScriptMetaData.java => scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala} (59%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelXlsReader.java rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResultResource.java => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelReader.scala (91%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/excel/StorageMultiExcelWriter.java => scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala} (58%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/io/IOClientFactory.java => scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala} (53%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/resultset/ResultRecord.java => scala/org/apache/linkis/storage/resultset/ResultMetaData.scala} (85%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultRecord.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetFactory.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetWriter.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/html/HtmlResultSet.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOMetaData.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IORecord.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/script/ScriptFsReader.java => scala/org/apache/linkis/storage/resultset/io/IOResultDeserializer.scala} (56%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSerializer.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSet.scala rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/label/LabelUpdateRequest.java => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/picture/PictureResultSet.scala (53%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/RequestEngineUnlock.java => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableMetaData.scala (70%) rename linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/PythonExecuteError.java => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala (71%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/resultset/table/TableMetaData.java => scala/org/apache/linkis/storage/resultset/table/TableResultSet.scala} (57%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSerializer.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSet.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsReader.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/LineMetaData.java => scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala} (60%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/PYScriptCompaction.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/QLScriptCompaction.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.scala rename linkis-commons/linkis-storage/src/main/{java/org/apache/linkis/storage/excel/ExcelFsWriter.java => scala/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.scala} (56%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala rename linkis-public-enhancements/linkis-pes-common/src/test/scala/org/apache/linkis/errorcode/common/LinkisErrorCodeTest.scala => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/PYScriptParser.scala (59%) rename linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/response/ScriptFromBMLResponseTest.scala => linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/QLScriptParser.scala (59%) create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ScalaScriptParser.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ShellScriptParser.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala create mode 100644 linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala delete mode 100644 linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/LineMetaDataTest.java create mode 100644 linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java delete mode 100644 linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/resultset/StorageResultSetWriterFactoryTest.java delete mode 100644 linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/source/ResultsetFileSourceTest.java delete mode 100644 linkis-commons/linkis-storage/src/test/resources/result-read-test.dolphin delete mode 100644 linkis-commons/linkis-storage/src/test/resources/storage-read-test.dolphin rename linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientImplTest.scala => linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala (72%) delete mode 100644 linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java rename linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/{GovernanceCommonConf.scala => GovernaceCommonConf.scala} (81%) create mode 100644 linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java rename linkis-computation-governance/{linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/LabelScoreServiceInstance.java => linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/utils/ECMCacheUtils.java} (53%) create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala rename linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/restful/ListenerTest.java => linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMMetricsService.scala (91%) create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMMetricsService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/test/java/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterServiceTest.java delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.java => linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/AbstractLogCache.java (50%) rename linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/listener/test/TestContextValue.java => linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/TimeLogCache.java (68%) delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala rename linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/{EngineConnPidCallback.scala => EngineConnIdentifierCallback.scala} (99%) create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnTimedCallback.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorCode.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/{service/common/metrics => am/converter}/MetricsConverter.java (97%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/{service/common => am}/label/LabelChecker.java (95%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/{service/common => am}/pointer/EMNodPointer.java (96%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/{service/common => am}/pointer/EngineNodePointer.java (96%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/{service/common => am}/pointer/NodePointer.java (96%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/{service/common => am}/pointer/NodePointerBuilder.java (94%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/DefaultECAvailableRule.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/DefaultNodeSelector.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/AvailableNodeSelectRule.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ConcurrencyNodeSelectRule.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/NewECMStandbyRule.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ScoreNodeSelectRule.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/TaskInfoNodeSelectRule.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/cache/ConfCacheRemoveBroadcastListener.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMRegisterService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/ECMOperateService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/EMRegisterService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/EMUnregisterService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/AbstractEngineService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineConnCanKillService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineOperateService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineRecycleService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineInfoService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineOperateService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineStopService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/EMUtils.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/LinkisUtils.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/utils/AMUtils.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/utils/DefaultRetryHandler.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/CanCreateECRes.java rename linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/listener/test/TestContextKey.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ConfigVo.java (51%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/conf/LabelManagerConf.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/NodeLabelRemoveService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/NodeLabelService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultResourceLabelService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultUserLabelService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/entity/LabelResourceMapping.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/kubernetes/KubernetesResourceRequester.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/parser/KubernetesResourceIdentifierParser.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnQueueInfo.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/message/RMMessageService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/restful/RMMonitorRest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/RequestResourceService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/ResourceLockService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/ChangeType.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/DriverAndKubernetesReqResourceService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/LabelResourceServiceImpl.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/ResourceLogService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/UserResourceService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/RMUtils.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/UserConfiguration.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/{postgresql => common}/EngineConnBmlResourceMapper.xml (69%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/mysql/EngineConnBmlResourceMapper.xml create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.java => scala/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.scala} (76%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoader.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/label/service/UserLabelService.java => scala/org/apache/linkis/engineplugin/server/localize/EngineConnBmlResourceGenerator.scala} (58%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.java => scala/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.scala} (73%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.java => scala/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.scala} (67%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.scala rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultMetaData.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHook.scala (84%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.java => scala/org/apache/linkis/manager/am/hook/AskEngineConnHookContext.scala} (78%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/DefaultNodeSelector.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/ECAvailableRule.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/selector/NodeSelector.java => scala/org/apache/linkis/manager/am/selector/NodeSelector.scala} (74%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/AvailableNodeSelectRule.scala rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineRecord.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ConcurrencyNodeSelectRule.scala (60%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/selector/rule/HotspotExclusionRule.java => scala/org/apache/linkis/manager/am/selector/rule/HotspotExclusionRule.scala} (53%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/NewECMStandbyRule.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/selector/rule/NodeSelectRule.java => scala/org/apache/linkis/manager/am/selector/rule/NodeSelectRule.scala} (81%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/OverLoadNodeSelectRule.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ScoreNodeSelectRule.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/TaskInfoNodeSelectRule.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/EMEngineService.java => scala/org/apache/linkis/manager/am/service/EMEngineService.scala} (52%) rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Parser.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/EngineService.scala (79%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/HeartbeatService.java => scala/org/apache/linkis/manager/am/service/HeartbeatService.scala} (85%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/cache/ConfCacheRemoveBroadcastListener.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMRegisterService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResource.java => scala/org/apache/linkis/manager/am/service/em/ECMOperateService.scala} (72%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/em/EMInfoService.java => scala/org/apache/linkis/manager/am/service/em/EMInfoService.scala} (63%) rename linkis-computation-governance/linkis-manager/{linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResourceInfo.java => linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/EMRegisterService.scala} (66%) rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClient.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/EMUnregisterService.scala (70%) rename linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSourceConfig.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/AbstractEngineService.scala (57%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnCanKillService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.scala rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Column.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineOperateService.scala (55%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineRecycleService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/engine/DefaultEngineSwitchService.java => scala/org/apache/linkis/manager/am/service/engine/DefaultEngineSwitchService.scala} (68%) rename linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/bean/ClientTestContextValue.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.scala (59%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/EngineService.java => scala/org/apache/linkis/manager/am/service/engine/EngineConnCanKillService.scala} (75%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/engine/EngineCreateService.java => scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala} (69%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineInfoService.scala rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableRecord.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineKillService.scala (67%) rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IORecord.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineOperateService.scala (67%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/engine/EngineRecycleService.java => scala/org/apache/linkis/manager/am/service/engine/EngineRecycleService.scala} (72%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/engine/EngineReuseService.java => scala/org/apache/linkis/manager/am/service/engine/EngineReuseService.scala} (69%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/service/engine/EngineSwitchService.java => scala/org/apache/linkis/manager/am/service/engine/EngineSwitchService.scala} (72%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/label/LabelManagerUtils.java => scala/org/apache/linkis/manager/label/LabelManagerUtils.scala} (61%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/score/LabelScoreServiceInstance.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/label/score/NodeLabelScorer.java => scala/org/apache/linkis/manager/label/score/NodeLabelScorer.scala} (60%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/label/service/NodeLabelAddService.java => scala/org/apache/linkis/manager/label/service/NodeLabelAddService.scala} (75%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelRemoveService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelService.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/label/service/ResourceLabelService.java => scala/org/apache/linkis/manager/label/service/ResourceLabelService.scala} (62%) rename linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/entity/enumeration/ExpireTypeTest.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/UserLabelService.scala (61%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultResourceLabelService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultUserLabelService.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/rm/external/kubernetes/KubernetesResourceIdentifier.java => scala/org/apache/linkis/manager/rm/entity/LabelResourceMap.scala} (51%) rename linkis-computation-governance/linkis-manager/{linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/OperateResponse.java => linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/entity/ResourceOperationType.scala} (81%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/message/RMMessageService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/rm/service/LabelResourceService.java => scala/org/apache/linkis/manager/rm/service/LabelResourceService.scala} (53%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/ResourceLockService.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/rm/service/ResourceManager.java => scala/org/apache/linkis/manager/rm/service/ResourceManager.scala} (54%) rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOMetaData.java => linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.scala (66%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/LabelResourceServiceImpl.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/UserResourceService.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/UserConfiguration.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/am/selector/ECAvailableRule.java => scala/org/apache/linkis/manager/service/common/label/LabelChecker.scala} (69%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/service/common/label/LabelFilter.java => scala/org/apache/linkis/manager/service/common/label/LabelFilter.scala} (73%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{java/org/apache/linkis/manager/service/common/label/ManagerLabelService.java => scala/org/apache/linkis/manager/service/common/label/ManagerLabelService.scala} (68%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/Scan.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/WebApplicationServer.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/engineplugin/server/dao/BaseDaoTest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/engineplugin/server/dao/EngineConnBmlResourceDaoTest.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/{ECResourceInfoLinkisUtilsTest.java => ECResourceInfoUtilsTest.java} (98%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/rm/external/dao/ExternalResourceProviderDaoTest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/resources/application.properties delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/resources/create.sql delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/resources/create_pg.sql delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/resources/linkis.properties create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/conf/ManagerCommonConf.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/Operator.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/OperatorFactoryImpl.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/RequestManagerUnlock.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/ResponseEngineLock.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/ResponseEngineUnlock.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/EMResourceRegisterRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/RegisterEMRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/RegisterEMResponse.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineAskAsyncResponse.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineAsyncResponse.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallbackToAM.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateError.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateSuccess.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/label/LabelReportRequest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResourceUsedProtocol.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResourceWithStatus.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResponseTaskRunningInfo.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/NotEnoughResource.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/RequestResourceAndWait.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/Operator.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/OperatorFactory.scala rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{java/org/apache/linkis/manager/common/protocol/RequestEngineLock.java => scala/org/apache/linkis/manager/common/protocol/EngineLock.scala} (56%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{java/org/apache/linkis/manager/common/protocol/OperateRequest.java => scala/org/apache/linkis/manager/common/protocol/OperateRequest.scala} (53%) rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptRecord.java => linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateResponse.scala (80%) rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/java/org/apache/linkis/manager/am/utils/RetryHandler.java => linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.scala} (58%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/EMResourceRegisterRequest.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/RegisterEMRequest.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/RegisterEMResponse.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineAsyncResponse.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.scala rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelAnalysisException.java => linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/label/LabelUpdateRequest.scala (65%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala rename linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/listener/execution/ExecutionTaskStatusListener.scala => linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceWithStatus.scala (70%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{java/org/apache/linkis/manager/common/protocol/EngineLock.java => scala/org/apache/linkis/manager/rm/ResourceInfo.scala} (82%) rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorCreator.java => linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResultResource.scala (79%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml rename linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/{postgresql => common}/LabelManagerMapper.xml (92%) rename linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/{postgresql => common}/ResourceManagerMapper.xml (85%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/mysql/ECResourceRecordMapper.xml delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/mysql/LabelManagerMapper.xml delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/mysql/ResourceManagerMapper.xml delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/postgresql/ECResourceRecordMapper.xml delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/ECResourceRecordMapperTest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/LabelManagerMapperTest.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/resources/create_pg.sql create mode 100644 linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/errorcode/FlinkErrorCodeSummary.java create mode 100644 linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/exception/ExecutorInitException.java create mode 100644 linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/exception/FlinkInitFailedException.java create mode 100644 linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/exception/JobExecutionException.java create mode 100644 linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/exception/SqlExecutionException.java rename linkis-engineconn-plugins/{shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.java => flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/exception/SqlParseException.java} (63%) create mode 100644 linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkManagerConcurrentExecutor.scala delete mode 100644 linkis-engineconn-plugins/io_file/src/test/java/executor/IoEngineConnExecutorTest.java delete mode 100644 linkis-engineconn-plugins/io_file/src/test/resources/testIoResult.dolphin delete mode 100644 linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/PythonEngineConnPlugin.java delete mode 100644 linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/conf/PythonEngineConfiguration.java delete mode 100644 linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/executor/PythonEngineConnExecutor.java delete mode 100644 linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/hook/PythonVersionEngineHook.java delete mode 100644 linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/utils/Kind.java create mode 100644 linkis-engineconn-plugins/python/src/main/scala/org/apache/linkis/manager/engineplugin/python/PythonEngineConnPlugin.scala create mode 100644 linkis-engineconn-plugins/python/src/main/scala/org/apache/linkis/manager/engineplugin/python/conf/PythonEngineConfiguration.scala rename linkis-engineconn-plugins/python/src/main/{java/org/apache/linkis/manager/engineplugin/python/exception/PythonSessionStartFailedExeception.java => scala/org/apache/linkis/manager/engineplugin/python/exception/NoSupportEngineException.scala} (68%) create mode 100644 linkis-engineconn-plugins/python/src/main/scala/org/apache/linkis/manager/engineplugin/python/exception/PythonSessionNullException.scala create mode 100644 linkis-engineconn-plugins/python/src/main/scala/org/apache/linkis/manager/engineplugin/python/executor/PythonEngineConnExecutor.scala create mode 100644 linkis-engineconn-plugins/python/src/main/scala/org/apache/linkis/manager/engineplugin/python/hook/PythonVersionEngineHook.scala rename linkis-engineconn-plugins/{shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.java => python/src/main/scala/org/apache/linkis/manager/engineplugin/python/launch/PythonProcessEngineConnLaunchBuilder.scala} (81%) create mode 100644 linkis-engineconn-plugins/python/src/main/scala/org/apache/linkis/manager/engineplugin/python/utils/Kind.scala create mode 100644 linkis-engineconn-plugins/python/src/main/scala/org/apache/linkis/manager/engineplugin/python/utils/State.scala delete mode 100644 linkis-engineconn-plugins/python/src/test/java/org/apache/linkis/manager/engineplugin/python/exception/TestNoSupportEngineException.java delete mode 100644 linkis-engineconn-plugins/python/src/test/java/org/apache/linkis/manager/engineplugin/python/executor/TestPythonEngineConnExecutor.java delete mode 100644 linkis-engineconn-plugins/python/src/test/java/org/apache/linkis/manager/engineplugin/python/factory/TestPythonEngineConnFactory.java rename linkis-engineconn-plugins/python/src/test/{java/org/apache/linkis/manager/engineplugin/python/TestPythonEngineConnPlugin.java => scala/org/apache/linkis/manager/engineplugin/python/TestPythonEngineConnPlugin.scala} (73%) rename linkis-engineconn-plugins/python/src/test/{java/org/apache/linkis/manager/engineplugin/python/conf/TestPythonEngineConfiguration.java => scala/org/apache/linkis/manager/engineplugin/python/conf/TestPythonEngineConfiguration.scala} (59%) create mode 100644 linkis-engineconn-plugins/python/src/test/scala/org/apache/linkis/manager/engineplugin/python/exception/TestNoSupportEngineException.scala create mode 100644 linkis-engineconn-plugins/python/src/test/scala/org/apache/linkis/manager/engineplugin/python/executor/TestPythonEngineConnExecutor.scala create mode 100644 linkis-engineconn-plugins/python/src/test/scala/org/apache/linkis/manager/engineplugin/python/factory/TestPythonEngineConnFactory.scala rename linkis-engineconn-plugins/python/src/test/{java/org/apache/linkis/manager/engineplugin/python/utils/TestKind.java => scala/org/apache/linkis/manager/engineplugin/python/utils/TestKind.scala} (66%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResponseTaskYarnResource.java => linkis-engineconn-plugins/python/src/test/scala/org/apache/linkis/manager/engineplugin/python/utils/TestState.scala (52%) delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/common/ShellEngineConnPluginConst.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/ShellCodeErrorException.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.java create mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.scala rename linkis-engineconn-plugins/{python/src/main/java/org/apache/linkis/manager/engineplugin/python/launch/PythonProcessEngineConnLaunchBuilder.java => shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.scala} (81%) rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/IExcelRowDeal.java => linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/common/ShellEnginePluginConst.scala (76%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnBmlResourceGenerator.java => linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.scala (68%) create mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.scala create mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala create mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.scala create mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala create mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala create mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.scala delete mode 100644 linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java rename linkis-engineconn-plugins/shell/src/test/{java/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.java => scala/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.scala} (75%) rename linkis-engineconn-plugins/shell/src/test/{java/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.java => scala/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.scala} (70%) rename linkis-engineconn-plugins/shell/src/test/{java/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.java => scala/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.scala} (67%) create mode 100644 linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.scala delete mode 100644 linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala delete mode 100644 linkis-engineconn-plugins/spark/src/test/resources/etltest.dolphin delete mode 100644 linkis-engineconn-plugins/spark/src/test/scala/org/apache/linkis/engineplugin/spark/datacalc/TestExcelCala.scala delete mode 100644 linkis-engineconn-plugins/spark/src/test/scala/org/apache/linkis/engineplugin/spark/datacalc/TestKafkaCala.scala delete mode 100644 linkis-engineconn-plugins/spark/src/test/scala/org/apache/linkis/engineplugin/spark/datacalc/TestRedisCalc.scala delete mode 100644 linkis-engineconn-plugins/spark/src/test/scala/org/apache/linkis/engineplugin/spark/datacalc/TestRocketmqCala.scala create mode 100644 linkis-engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/client/RemoteClientHolder.scala rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/AvailableResource.java => linkis-engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/exception/DataSourceRpcErrorException.scala (69%) create mode 100644 linkis-engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/params/ConnectParamsResolver.scala create mode 100644 linkis-engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/params/SqoopDataSourceParamsResolver.scala delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/TrinoEngineConnPlugin.java delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/conf/TrinoConfiguration.java delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/conf/TrinoEngineConfig.java delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/exception/TrinoClientException.java delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/exception/TrinoGrantmaException.java delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/exception/TrinoModifySchemaException.java delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/exception/TrinoStateInvalidException.java delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/executor/TrinoEngineConnExecutor.java rename {linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/java/org/apache/linkis/engineconn/common => linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino}/password/CommandPasswordCallback.java (93%) rename {linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/java/org/apache/linkis/engineconn/common => linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino}/password/StaticPasswordCallback.java (95%) delete mode 100644 linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/utils/TrinoCode.java create mode 100644 linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/TrinoEngineConnPlugin.scala rename linkis-engineconn-plugins/trino/src/main/{java/org/apache/linkis/engineplugin/trino/builder/TrinoProcessEngineConnLaunchBuilder.java => scala/org/apache/linkis/engineplugin/trino/builder/TrinoProcessEngineConnLaunchBuilder.scala} (67%) create mode 100644 linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/conf/TrinoConfiguration.scala create mode 100644 linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/conf/TrinoEngineConfig.scala create mode 100644 linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/exception/TrinoException.scala create mode 100644 linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/executor/TrinoEngineConnExecutor.scala create mode 100644 linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/utils/TrinoCode.scala rename linkis-engineconn-plugins/trino/src/main/{java/org/apache/linkis/engineplugin/trino/utils/TrinoSQLHook.java => scala/org/apache/linkis/engineplugin/trino/utils/TrinoSQLHook.scala} (72%) create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ApplicationConfiguration.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceClear.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/TaskArchiveClear.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/JobMonitorUtils.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/index/JobIndexHitEvent.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/index/JobIndexRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/index/JobIndexSender.scala delete mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/UserLabelAlertUtils.scala rename linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/{postgresql => common}/BmlProjectMapper.xml (63%) rename linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/{mysql => common}/ResourceMapper.xml (82%) rename linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/{postgresql => common}/TaskMapper.xml (91%) rename linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/{postgresql => common}/VersionMapper.xml (74%) delete mode 100644 linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/mysql/BmlProjectMapper.xml delete mode 100644 linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/mysql/TaskMapper.xml delete mode 100644 linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/mysql/VersionMapper.xml delete mode 100644 linkis-public-enhancements/linkis-bml-server/src/main/resources/mapper/postgresql/ResourceMapper.xml delete mode 100644 linkis-public-enhancements/linkis-bml-server/src/test/resources/create_pg.sql rename linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/{mysql => common}/AcrossClusterRuleMapper.xml (98%) rename linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/{mysql => common}/ConfigKeyLimitForUserMapper.xml (100%) rename linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/{mysql => common}/ConfigMapper.xml (98%) rename linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/{postgresql/UserTenantMapper.xml => common/DepartmentTenantMapper.xml} (53%) rename linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/{mysql => common}/TemplateConfigKeyMapper.xml (100%) rename linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/{mysql => common}/UserIpMapper.xml (93%) rename linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/{mysql => common}/UserTenantMapper.xml (97%) delete mode 100644 linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/postgresql/AcrossClusterRuleMapper.xml delete mode 100644 linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/postgresql/ConfigKeyLimitForUserMapper.xml delete mode 100644 linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/postgresql/ConfigMapper.xml delete mode 100644 linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/postgresql/TemplateConfigKeyMapper.xml delete mode 100644 linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/postgresql/UserIpMapper.xml delete mode 100644 linkis-public-enhancements/linkis-configuration/src/test/resources/create_pg.sql rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineConnCanKillService.java => linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/conf/CSConfiguration.java (67%) rename linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/{mysql => common}/contextHistoryMapper.xml (86%) rename linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/{mysql => common}/contextIDListenerMapper.xml (83%) rename linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/{mysql => common}/contextIDMapper.xml (98%) rename linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/{mysql => common}/contextMapMapper.xml (98%) delete mode 100644 linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/postgresql/contextHistoryMapper.xml delete mode 100644 linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/postgresql/contextIDListenerMapper.xml delete mode 100644 linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/postgresql/contextIDMapper.xml delete mode 100644 linkis-public-enhancements/linkis-cs-server/src/main/resources/mapper/postgresql/contextMapMapper.xml delete mode 100644 linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/dao/ContextMapMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-cs-server/src/test/resources/create_pg.sql rename linkis-public-enhancements/linkis-instance-label-server/src/main/resources/mapper/{mysql => common}/InsLabelRelationMapper.xml (76%) rename linkis-public-enhancements/linkis-instance-label-server/src/main/resources/mapper/{mysql => common}/InstanceInfoMapper.xml (74%) rename linkis-public-enhancements/linkis-instance-label-server/src/main/resources/mapper/{mysql => common}/InstanceLabelMapper.xml (70%) delete mode 100644 linkis-public-enhancements/linkis-instance-label-server/src/main/resources/mapper/postgresql/InsLabelRelationMapper.xml delete mode 100644 linkis-public-enhancements/linkis-instance-label-server/src/main/resources/mapper/postgresql/InstanceInfoMapper.xml delete mode 100644 linkis-public-enhancements/linkis-instance-label-server/src/main/resources/mapper/postgresql/InstanceLabelMapper.xml delete mode 100644 linkis-public-enhancements/linkis-instance-label-server/src/test/resources/create_pg.sql create mode 100644 linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/JobhistoryUtils.java delete mode 100644 linkis-public-enhancements/linkis-jobhistory/src/test/java/org/apache/linkis/jobhistory/dao/JobDetailMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-jobhistory/src/test/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-jobhistory/src/test/resources/create_pg.sql delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/bml/client/errorcode/BmlClientErrorCodeSummaryTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/builder/ContextClientFactoryTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/builder/HttpContextClientConfigTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/errorcode/CsClientErrorCodeSummaryTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/Test.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/bean/ClientTestContextKey.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/bean/ResourceValueBean.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/listener/CommonContextKeyListener.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/no_context_search/TestClear.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/restful/RestfulTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/service/TestInfo.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/service/TestRemove.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/service/TestSearchService.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/test_multiuser/TestChangeContext.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/test_multiuser/TestCreateContext.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/utils/ContextClientConfTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/utils/ContextServiceUtilsTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/utils/SerializeHelperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/datasource/client/TestDataSourceClient.scala delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/datasource/client/TestHiveClient.scala delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/datasource/client/TestMysqlClient.scala delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/datasource/client/config/DatasourceClientConfigTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/datasource/client/exception/DataSourceClientBuilderExceptionTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/datasource/client/imp/LinkisDataSourceRemoteClientTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/errorcode/client/ClientConfigurationTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/errorcode/client/ErrorCodeClientBuilderTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/errorcode/client/LinkisErrorCodeClientTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/errorcode/client/handler/ErrorCodeHandlerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandlerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/errorcode/client/manager/LinkisErrorCodeManagerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/errorcode/client/synchronizer/LinkisErrorCodeSynchronizerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/errorcode/client/action/ErrorCodeActionTest.scala delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/action/OpenScriptFromBMLActionTest.scala delete mode 100644 linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/conf/WorkspaceClientConfTest.scala create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala rename linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/CSVFsWriter.java => linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala (60%) delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/entity/enumeration/ContextScopeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/entity/enumeration/ContextTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/entity/enumeration/DBTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/entity/enumeration/WorkTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/protocol/ContextHistoryTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/protocol/ContextIDTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/protocol/ContextKeyTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/protocol/ContextKeyValueTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/protocol/ContextValueTypeTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/helper/ContextSerializationHelperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/impl/context/CombinedNodeIDContextIDSerializerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/impl/context/CommonContextKeySerializerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/impl/value/LinkisBMLResourceSerializerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/impl/value/data/CSResultDataSerializerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/impl/value/data/LinkisJobDataSerializerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/impl/value/metadata/CSTableSerializerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/impl/value/object/CSFlowInfosSerializerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/serialize/test/ContextSerializationHelperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/utils/CSCommonUtilsTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/common/utils/CSHighAvailableUtilsTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/listener/test/TestContextID.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/listener/test/TestContextKeyValue.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/errorcode/common/CommonConfTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/metadata/query/common/MdmConfigurationTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/metadata/query/common/cache/CacheConfigurationTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/metadata/query/common/cache/ConnCacheManagerTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/metadata/query/common/exception/MetaMethodInvokeExceptionTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/metadata/query/common/exception/MetaRuntimeExceptionTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/udf/excepiton/UDFExceptionTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/udf/utils/ConstantVarTest.java rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/ConfigurationConfigKeyMapper.xml (68%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/DatasourceAccessMapper.xml (91%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/DatasourceEnvMapper.xml (90%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/DatasourceTypeKeyMapper.xml (92%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/DatasourceTypeMapper.xml (86%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/GatewayAuthTokenMapper.xml (90%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{postgresql => common}/PsErrorCodeMapper.xml (86%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/RmExternalResourceProviderMapper.xml (86%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{postgresql/UdfManagerMapper.xml => common/UdfBaseInfoMapper.xml} (61%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/UdfManagerMapper.xml (93%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/UdfTreeMapper.xml (88%) rename linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/{mysql => common}/VarMapper.xml (91%) delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/mysql/PsErrorCodeMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/ConfigurationConfigKeyMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/DatasourceAccessMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/DatasourceEnvMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/DatasourceTypeKeyMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/DatasourceTypeMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/GatewayAuthTokenMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/RmExternalResourceProviderMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/UdfTreeMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/resources/mapper/postgresql/VarMapper.xml delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/Scan.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/WebApplicationServer.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/BaseDaoTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/ConfigurationConfigKeyMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/ConfigurationConfigValueMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/ConfigurationKeyEngineRelationMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/DatasourceAccessMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/DatasourceEnvMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/DatasourceTypeMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/EngineConnPluginBmlResourcesMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/GatewayAuthTokenMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/PsErrorCodeMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/RmExternalResourceProviderMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/UdfManagerMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/UdfTreeMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/ConfigurationTemplateRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/DatasourceAccessRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/DatasourceEnvRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeKeyRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/ErrorCodeRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/MvcUtils.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/RmExternalResourceProviderRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/DatasourceAccessServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/DatasourceEnvServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/DatasourceTypeKeyServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/DatasourceTypeServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/ErrorCodeServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/GatewayAuthTokenServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/RmExternalResourceProviderServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/UdfManagerServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/service/UdfTreeServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/errorcode/server/Scan.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/errorcode/server/WebApplicationServer.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/errorcode/server/dao/BaseDaoTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/errorcode/server/dao/ErrorCodeMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/variable/Scan.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/variable/WebApplicationServer.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/variable/dao/BaseDaoTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/variable/dao/VarMapperTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/variable/restful/MvcUtils.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/variable/restful/api/VariableRestfulApiTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/java/org/apache/linkis/variable/service/VariableServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/resources/basedata_manager_create.sql delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/resources/basedata_manager_create_pg.sql delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/resources/error_code_create.sql delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/resources/error_code_create_pg.sql delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/resources/info.text delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/resources/variable_create.sql delete mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/test/resources/variable_create_pg.sql delete mode 100644 linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/create_pg.sql create mode 100644 linkis-public-enhancements/linkis-ps-common-lock/src/test/resources/data.sql rename linkis-public-enhancements/{linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/CgManagerLabelMapperTest.java => linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java} (50%) create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java rename linkis-public-enhancements/{linkis-pes-publicservice/src/test/java/org/apache/linkis/basedatamanager/server/dao/DatasourceTypeKeyMapperTest.java => linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java} (54%) create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml rename linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/{postgresql => common}/UDFDao.xml (76%) rename linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/{postgresql => common}/UDFVersionDao.xml (85%) delete mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/mysql/UDFDao.xml delete mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/mysql/UDFVersionDao.xml create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java delete mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/resources/create_pg.sql create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/resources/data.sql create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/ECMRequestGatewayParser.scala diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/FSFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/FSFactory.java deleted file mode 100644 index 34a92ead85..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/FSFactory.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.factory.BuildFactory; -import org.apache.linkis.storage.utils.StorageConfiguration; -import org.apache.linkis.storage.utils.StorageUtils; - -import java.text.MessageFormat; -import java.util.Map; - -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_FILE; - -public class FSFactory { - private static final Map buildClasses = - StorageUtils.loadClass( - StorageConfiguration.STORAGE_BUILD_FS_CLASSES.getValue(), t -> t.fsName()); - - public static BuildFactory getBuildFactory(String fsName) { - if (!buildClasses.containsKey(fsName)) { - throw new StorageWarnException( - UNSUPPORTED_FILE.getErrorCode(), - MessageFormat.format(UNSUPPORTED_FILE.getErrorDesc(), fsName)); - } - return buildClasses.get(fsName); - } - - public static Fs getFs(String fsType, String proxyUser) { - String user = StorageUtils.getJvmUser(); - return getBuildFactory(fsType).getFs(user, proxyUser); - } - - public static Fs getFs(String fsType) { - String user = StorageUtils.getJvmUser(); - return getBuildFactory(fsType).getFs(user, user); - } - - /** - * 1. If this machine has shared storage, the file:// type FS obtained here is the FS of the - * process user. 2. If this machine does not have shared storage, then the file:// type FS - * obtained is the proxy to the Remote (shared storage machine root) FS. 3. If it is HDFS, it - * returns the FS of the process user. 1、如果这台机器装有共享存储则这里获得的file://类型的FS为该进程用户的FS - * 2、如果这台机器没有共享存储则获得的file://类型的FS为代理到Remote（共享存储机器root）的FS 3、如果是HDFS则返回的就是该进程用户的FS - * - * @param fsPath - * @return - */ - public static Fs getFs(FsPath fsPath) { - return getFs(fsPath.getFsType()); - } - - /** - * 1. If the process user is passed and the proxy user and the process user are consistent, the - * file:// type FS is the FS of the process user (the shared storage exists) 2. If the process - * user is passed and the proxy user and the process user are consistent and there is no shared - * storage, the file:// type FS is the proxy to the remote (shared storage machine root) FS 3. If - * the passed proxy user and process user are consistent, the hdfs type is the FS of the process - * user. 4. If the proxy user and the process user are inconsistent, the hdfs type is the FS after - * the proxy. - * - * @param fsPath - * @param proxyUser - * @return - */ - public static Fs getFsByProxyUser(FsPath fsPath, String proxyUser) { - return getFs(fsPath.getFsType(), proxyUser); - } - - public Fs getFSByLabel(String fs, String label) { - String user = StorageUtils.getJvmUser(); - return getBuildFactory(fs).getFs(user, user, label); - } - - public Fs getFSByLabelAndUser(String fs, String label, String proxy) { - String user = StorageUtils.getJvmUser(); - return getBuildFactory(fs).getFs(user, proxy, label); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java deleted file mode 100644 index 7fb5c1b4ac..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.conf; - -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.utils.ByteTimeUtils; - -import org.apache.commons.lang3.StringUtils; - -public class LinkisStorageConf { - private static final Object CONF_LOCK = new Object(); - - public static final String DOLPHIN = "dolphin"; - - public static final String PARQUET = "parquet"; - - public static final String PARQUET_FILE_SUFFIX = ".parquet"; - - public static final String ORC = "orc"; - - public static final String ORC_FILE_SUFFIX = ".orc"; - - public static final String HDFS_FILE_SYSTEM_REST_ERRS = - CommonVars.apply( - "wds.linkis.hdfs.rest.errs", - ".*Filesystem closed.*|.*Failed to find any Kerberos tgt.*") - .getValue(); - - public static final String ROW_BYTE_MAX_LEN_STR = - CommonVars.apply("wds.linkis.resultset.row.max.str", "2m").getValue(); - - public static final String ENGINE_RESULT_TYPE = - CommonVars.apply("linkis.engine.resultSet.type", DOLPHIN, "Result type").getValue(); - - public static final long ROW_BYTE_MAX_LEN = ByteTimeUtils.byteStringAsBytes(ROW_BYTE_MAX_LEN_STR); - - public static final String FILE_TYPE = - CommonVars.apply( - "wds.linkis.storage.file.type", - "dolphin,sql,scala,py,hql,python,out,log,text,txt,sh,jdbc,ngql,psql,fql,tsql" - + "," - + PARQUET - + "," - + ORC) - .getValue(); - - private static volatile String[] fileTypeArr = null; - - private static String[] fileTypeArrParser(String fileType) { - if (StringUtils.isBlank(fileType)) { - return new String[0]; - } else { - return fileType.split(","); - } - } - - public static String[] getFileTypeArr() { - if (fileTypeArr == null) { - synchronized (CONF_LOCK) { - if (fileTypeArr == null) { - fileTypeArr = fileTypeArrParser(FILE_TYPE); - } - } - } - return fileTypeArr; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java deleted file mode 100644 index d98be40337..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/csv/StorageCSVWriter.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.csv; - -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.domain.DataType; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class StorageCSVWriter extends CSVFsWriter { - private static final Logger logger = LoggerFactory.getLogger(StorageCSVWriter.class); - - private final String charset; - private final String separator; - private final boolean quoteRetouchEnable; - private final OutputStream outputStream; - - private final String delimiter; - private final StringBuilder buffer; - - public StorageCSVWriter( - String charset, String separator, boolean quoteRetouchEnable, OutputStream outputStream) { - this.charset = charset; - this.separator = separator; - this.quoteRetouchEnable = quoteRetouchEnable; - this.outputStream = outputStream; - - if (StringUtils.isBlank(separator)) { - this.delimiter = "\t"; - } else { - switch (separator) { - case "t": - this.delimiter = "\t"; - break; - default: - this.delimiter = separator; - break; - } - } - this.buffer = new StringBuilder(50000); - } - - @Override - public String getCharset() { - return charset; - } - - @Override - public String getSeparator() { - return separator; - } - - @Override - public boolean isQuoteRetouchEnable() { - return quoteRetouchEnable; - } - - @Override - public void addMetaData(MetaData metaData) throws IOException { - Column[] columns = ((TableMetaData) metaData).getColumns(); - String[] head = Stream.of(columns).map(Column::getColumnName).toArray(String[]::new); - write(head); - } - - private String compact(String[] row) { - String quotationMarks = "\""; - String dealNewlineSymbolMarks = "\n"; - StringBuilder rowBuilder = new StringBuilder(); - for (String value : row) { - String decoratedValue = value; - if (StringUtils.isNotBlank(value)) { - if (quoteRetouchEnable) { - decoratedValue = quotationMarks + value.replaceAll(quotationMarks, "") + quotationMarks; - } - decoratedValue = decoratedValue.replaceAll(dealNewlineSymbolMarks, " "); - logger.debug("decorateValue with input: {} output: {} ", value, decoratedValue); - } - rowBuilder.append(decoratedValue).append(delimiter); - } - if (rowBuilder.length() > 0 && rowBuilder.toString().endsWith(delimiter)) { - int index = rowBuilder.lastIndexOf(delimiter); - rowBuilder.delete(index, index + delimiter.length()); - } - rowBuilder.append("\n"); - if (logger.isDebugEnabled()) { - logger.debug("delimiter:" + delimiter); - } - return rowBuilder.toString(); - } - - private void write(String[] row) throws IOException { - String content = compact(row); - if (buffer.length() + content.length() > 49500) { - IOUtils.write(buffer.toString().getBytes(charset), outputStream); - buffer.setLength(0); - } - buffer.append(content); - } - - @Override - public void addRecord(Record record) throws IOException { - Object[] rows = ((TableRecord) record).row; - String[] body = - Stream.of(rows).map(dataType -> DataType.valueToString(dataType)).toArray(String[]::new); - write(body); - } - - @Override - public void flush() throws IOException { - IOUtils.write(buffer.toString().getBytes(charset), outputStream); - buffer.setLength(0); - } - - @Override - public void close() throws IOException { - flush(); - IOUtils.closeQuietly(outputStream); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java deleted file mode 100644 index 6808f693ec..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/DataType.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.domain; - -import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; -import java.util.Optional; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.domain.Dolphin.LINKIS_NULL; - -public enum DataType { - NullType("void", 0), - StringType("string", 12), - BooleanType("boolean", 16), - TinyIntType("tinyint", -6), - ShortIntType("short", 5), - IntType("int", 4), - LongType("long", -5), - BigIntType("bigint", -5), - FloatType("float", 6), - DoubleType("double", 8), - CharType("char", 1), - VarcharType("varchar", 12), - DateType("date", 91), - TimestampType("timestamp", 93), - BinaryType("binary", -2), - DecimalType("decimal", 3), - ArrayType("array", 2003), - MapType("map", 2000), - ListType("list", 2001), - StructType("struct", 2002), - BigDecimalType("bigdecimal", 3); - - private final String typeName; - private final int javaSQLType; - - DataType(String typeName, int javaSQLType) { - this.typeName = typeName; - this.javaSQLType = javaSQLType; - } - - private static Logger logger = LoggerFactory.getLogger(DataType.class); - - public static final String NULL_VALUE = "NULL"; - public static final String LOWCASE_NULL_VALUE = "null"; - - // TODO Change to fine-grained regular expressions(改为精细化正则表达式) - public static final Pattern DECIMAL_REGEX = - Pattern.compile("^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)"); - - public static final Pattern SHORT_REGEX = Pattern.compile("^short.*"); - public static final Pattern INT_REGEX = Pattern.compile("^int.*"); - public static final Pattern LONG_REGEX = Pattern.compile("^long.*"); - public static final Pattern BIGINT_REGEX = Pattern.compile("^bigint.*"); - public static final Pattern FLOAT_REGEX = Pattern.compile("^float.*"); - public static final Pattern DOUBLE_REGEX = Pattern.compile("^double.*"); - - public static final Pattern VARCHAR_REGEX = Pattern.compile("^varchar.*"); - public static final Pattern CHAR_REGEX = Pattern.compile("^char.*"); - - public static final Pattern ARRAY_REGEX = Pattern.compile("array.*"); - - public static final Pattern MAP_REGEX = Pattern.compile("map.*"); - - public static final Pattern LIST_REGEX = Pattern.compile("list.*"); - - public static final Pattern STRUCT_REGEX = Pattern.compile("struct.*"); - - public static DataType toDataType(String dataType) { - if (dataType.equals("void") || dataType.equals("null")) { - return DataType.NullType; - } else if (dataType.equals("string")) { - return DataType.StringType; - } else if (dataType.equals("boolean")) { - return DataType.BooleanType; - } else if (SHORT_REGEX.matcher(dataType).matches()) { - return DataType.ShortIntType; - } else if (LONG_REGEX.matcher(dataType).matches()) { - return DataType.LongType; - } else if (BIGINT_REGEX.matcher(dataType).matches()) { - return DataType.BigIntType; - } else if (INT_REGEX.matcher(dataType).matches() - || dataType.equals("integer") - || dataType.equals("smallint")) { - return DataType.IntType; - } else if (FLOAT_REGEX.matcher(dataType).matches()) { - return DataType.FloatType; - } else if (DOUBLE_REGEX.matcher(dataType).matches()) { - return DataType.DoubleType; - } else if (VARCHAR_REGEX.matcher(dataType).matches()) { - return DataType.VarcharType; - } else if (CHAR_REGEX.matcher(dataType).matches()) { - return DataType.CharType; - } else if (dataType.equals("date")) { - return DataType.DateType; - } else if (dataType.equals("timestamp")) { - return DataType.TimestampType; - } else if (dataType.equals("binary")) { - return DataType.BinaryType; - } else if (dataType.equals("decimal") || DECIMAL_REGEX.matcher(dataType).matches()) { - return DataType.DecimalType; - } else if (ARRAY_REGEX.matcher(dataType).matches()) { - return DataType.ArrayType; - } else if (MAP_REGEX.matcher(dataType).matches()) { - return DataType.MapType; - } else if (LIST_REGEX.matcher(dataType).matches()) { - return DataType.ListType; - } else if (STRUCT_REGEX.matcher(dataType).matches()) { - return DataType.StructType; - } else { - return DataType.StringType; - } - } - - public static Object toValue(DataType dataType, String value) { - - Object result = null; - if (isLinkisNull(value)) { - return result; - } - try { - switch (dataType) { - case NullType: - result = null; - break; - case StringType: - case CharType: - case VarcharType: - case StructType: - case ListType: - case ArrayType: - case MapType: - result = value; - break; - case BooleanType: - result = isNumberNull(value) ? null : Boolean.valueOf(value); - break; - case ShortIntType: - result = isNumberNull(value) ? null : Short.valueOf(value); - break; - case IntType: - result = isNumberNull(value) ? null : Integer.valueOf(value); - break; - case LongType: - case BigIntType: - result = isNumberNull(value) ? null : Long.valueOf(value); - break; - case FloatType: - result = isNumberNull(value) ? null : Float.valueOf(value); - break; - case DoubleType: - result = isNumberNull(value) ? null : Double.valueOf(value); - break; - case DecimalType: - result = isNumberNull(value) ? null : new BigDecimal(value); - break; - case DateType: - result = isNumberNull(value) ? null : Date.valueOf(value); - break; - case TimestampType: - result = - isNumberNull(value) - ? null - : Optional.of(value) - .map(Timestamp::valueOf) - .map(Timestamp::toString) - .map(s -> s.endsWith(".0") ? s.substring(0, s.length() - 2) : s) - .orElse(null); - break; - case BinaryType: - result = isNull(value) ? null : value.getBytes(); - break; - default: - result = value; - } - } catch (Exception e) { - logger.debug("Failed to {} switch to dataType:", value, e); - result = value; - } - return result; - } - - public static boolean isLinkisNull(String value) { - return value == null || value.equals(LINKIS_NULL); - } - - public static boolean isNull(String value) { - return value == null || value.equals(NULL_VALUE) || value.trim().equals(""); - } - - public static boolean isNumberNull(String value) { - return value == null || value.equalsIgnoreCase(NULL_VALUE) || value.trim().equals(""); - } - - public static String valueToString(Object value) { - if (value == null) { - return LOWCASE_NULL_VALUE; - } else if (value instanceof BigDecimal) { - return ((BigDecimal) value).toPlainString(); - } else { - return value.toString(); - } - } - - public String getTypeName() { - return typeName; - } - - public int getJavaSQLType() { - return javaSQLType; - } - - @Override - public String toString() { - return typeName; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java deleted file mode 100644 index 35c71295e4..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/Dolphin.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.domain; - -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.utils.StorageConfiguration; -import org.apache.linkis.storage.utils.StorageUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.nio.charset.Charset; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.FAILED_TO_READ_INTEGER; - -public class Dolphin { - private static final Logger logger = LoggerFactory.getLogger(Dolphin.class); - - public static final Charset CHAR_SET = - Charset.forName(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue()); - public static final String MAGIC = "dolphin"; - - public static byte[] MAGIC_BYTES = new byte[0]; - - static { - try { - MAGIC_BYTES = MAGIC.getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue()); - } catch (UnsupportedEncodingException e) { - logger.warn("Dolphin getBytes failed", e); - } - } - - public static final int MAGIC_LEN = MAGIC_BYTES.length; - - public static final String DOLPHIN_FILE_SUFFIX = ".dolphin"; - - public static final String COL_SPLIT = ","; - public static final byte[] COL_SPLIT_BYTES = COL_SPLIT.getBytes(Charset.forName("utf-8")); - public static final int COL_SPLIT_LEN = COL_SPLIT_BYTES.length; - - public static final String NULL = "NULL"; - public static final byte[] NULL_BYTES = "NULL".getBytes(Charset.forName("utf-8")); - - public static final String LINKIS_NULL = "LINKIS_NULL"; - public static final byte[] LINKIS_NULL_BYTES = LINKIS_NULL.getBytes(Charset.forName("utf-8")); - - public static final int INT_LEN = 10; - - public static final int FILE_EMPTY = 31; - - public static byte[] getBytes(Object value) { - return value.toString().getBytes(CHAR_SET); - } - - /** - * Convert a bytes array to a String content 将bytes数组转换为String内容 - * - * @param bytes - * @param start - * @param len - * @return - */ - public static String getString(byte[] bytes, int start, int len) { - return new String(bytes, start, len, Dolphin.CHAR_SET); - } - - public static String toStringValue(String value) { - if (LINKIS_NULL.equals(value)) { - return NULL; - } else { - return value; - } - } - - /** - * Read an integer value that converts the array to a byte of length 10 bytes - * 读取整数值，该值为将数组转换为10字节长度的byte - * - * @param inputStream - * @return - * @throws IOException - */ - public static int readInt(InputStream inputStream) throws IOException { - byte[] bytes = new byte[INT_LEN + 1]; - if (StorageUtils.readBytes(inputStream, bytes, INT_LEN) != INT_LEN) { - throw new StorageWarnException( - FAILED_TO_READ_INTEGER.getErrorCode(), FAILED_TO_READ_INTEGER.getErrorDesc()); - } - return Integer.parseInt(getString(bytes, 0, INT_LEN)); - } - - /** - * Print integers at a fixed length(将整数按固定长度打印) - * - * @param value - * @return - */ - public static byte[] getIntBytes(int value) { - String str = Integer.toString(value); - StringBuilder res = new StringBuilder(); - for (int i = 0; i < INT_LEN - str.length(); i++) { - res.append("0"); - } - res.append(str); - return Dolphin.getBytes(res.toString()); - } - - public static String getType(InputStream inputStream) throws IOException { - byte[] bytes = new byte[100]; - int len = StorageUtils.readBytes(inputStream, bytes, Dolphin.MAGIC_LEN + INT_LEN); - if (len == -1) return null; - return getType(Dolphin.getString(bytes, 0, len)); - } - - public static String getType(String content) { - if (content.length() < MAGIC.length() || !content.substring(0, MAGIC.length()).equals(MAGIC)) { - throw new RuntimeException( - "File header type must be dolphin, content: " + content + " is not"); - } - return Integer.toString( - Integer.parseInt(content.substring(MAGIC.length(), MAGIC.length() + INT_LEN))); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntity.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntity.java deleted file mode 100644 index c1e16e223a..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntity.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.domain; - -/** - * Engine unique Id(engine唯一的Id) - * - *

Fs type(fs类型) - * - *

Create a user to start the corresponding jvm user(创建用户为对应启动的jvm用户) - * - *

Proxy user(代理用户) - * - *

client Ip for whitelist control(ip用于白名单控制) - * - *

Method name called(调用的方法名) - * - *

Method parameter(方法参数) - */ -public class MethodEntity { - private long id; - private String fsType; - private String creatorUser; - private String proxyUser; - private String clientIp; - private String methodName; - private Object[] params; - - public MethodEntity( - long id, - String fsType, - String creatorUser, - String proxyUser, - String clientIp, - String methodName, - Object[] params) { - this.id = id; - this.fsType = fsType; - this.creatorUser = creatorUser; - this.proxyUser = proxyUser; - this.clientIp = clientIp; - this.methodName = methodName; - this.params = params; - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getFsType() { - return fsType; - } - - public void setFsType(String fsType) { - this.fsType = fsType; - } - - public String getCreatorUser() { - return creatorUser; - } - - public void setCreatorUser(String creatorUser) { - this.creatorUser = creatorUser; - } - - public String getProxyUser() { - return proxyUser; - } - - public void setProxyUser(String proxyUser) { - this.proxyUser = proxyUser; - } - - public String getClientIp() { - return clientIp; - } - - public void setClientIp(String clientIp) { - this.clientIp = clientIp; - } - - public String getMethodName() { - return methodName; - } - - public void setMethodName(String methodName) { - this.methodName = methodName; - } - - public Object[] getParams() { - return params; - } - - public void setParams(Object[] params) { - this.params = params; - } - - @Override - public String toString() { - return "id:" - + id - + ", methodName:" - + methodName - + ", fsType:" - + fsType - + ", creatorUser:" - + creatorUser - + ", proxyUser:" - + proxyUser - + ", clientIp:" - + clientIp; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntitySerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntitySerializer.java deleted file mode 100644 index 777b756a7f..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/domain/MethodEntitySerializer.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.domain; - -import java.lang.reflect.Type; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -/** - * Serialize MethodEntity to code 序列化MethodEntity为code - * - *

Serialized to code as a MethodEntity object 序列化为code为MethodEntity对象 - * - *

Serialize a java object as a string 序列化java对象为字符串 - * - *

Deserialize a string into a java object 将字符串解序列化为java对象 - */ -public class MethodEntitySerializer { - - private static final Gson gson = - new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); - - /** - * Serialized to code as a MethodEntity object 序列化为code为MethodEntity对象 - * - * @param code - * @return - */ - public static MethodEntity deserializer(String code) { - return gson.fromJson(code, MethodEntity.class); - } - - /** - * Serialize MethodEntity to code 序列化MethodEntity为code - * - * @param methodEntity - * @return - */ - public static String serializer(MethodEntity methodEntity) { - return gson.toJson(methodEntity); - } - - /** - * Serialize a java object as a string 序列化java对象为字符串 - * - * @param value - * @return - */ - public static String serializerJavaObject(Object value) { - return gson.toJson(value); - } - - /** - * Deserialize a string into a java object 将字符串解序列化为java对象 - * - * @param json - * @param classType - * @param - * @return - */ - public static T deserializerToJavaObject(String json, Class classType) { - return gson.fromJson(json, classType); - } - - public static T deserializerToJavaObject(String json, Type oType) { - return gson.fromJson(json, oType); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java index 9ca3425837..7187f72ce5 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/errorcode/LinkisStorageErrorCodeSummary.java @@ -37,6 +37,14 @@ public enum LinkisStorageErrorCodeSummary implements LinkisErrorCode { 52004, "You must register IOMethodInterceptorCreator before you can use proxy mode.(必须先注册IOMethodInterceptorCreator，才能使用代理模式)"), UNSUPPORTED_OPEN_FILE_TYPE(54001, "Unsupported open file type(不支持打开的文件类型)"), + + RESULT_COL_LENGTH(52003, "Col value length {0} exceed limit {1}"), + + RESULT_COLUMN_INDEX_OUT_OF_BOUNDS(52004, "Column index value {0} exceed limit {1}"), + + RESULT_ROW_LENGTH(520034, "Row value length {0} exceed limit {1}"), + + RESULT_COL_SIZE(520035, "Col size length {0} exceed limit {1}"), INVALID_CUSTOM_PARAMETER(65000, "Invalid custom parameter(不合法的自定义参数)"); /** 错误码 */ diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelXlsReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelXlsReader.java deleted file mode 100644 index 98df7421f3..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelXlsReader.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.excel; - -import org.apache.poi.hssf.eventusermodel.*; -import org.apache.poi.hssf.eventusermodel.EventWorkbookBuilder.SheetRecordCollectingListener; -import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord; -import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord; -import org.apache.poi.hssf.model.HSSFFormulaParser; -import org.apache.poi.hssf.record.*; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ExcelXlsReader implements HSSFListener { - private static final Logger logger = LoggerFactory.getLogger(ExcelXlsReader.class); - - private int minColumns = -1; - - private POIFSFileSystem fs; - - private InputStream inputStream; - - private int lastRowNumber; - - private int lastColumnNumber; - - /** Should we output the formula, or the value it has? */ - private boolean outputFormulaValues = true; - - /** For parsing Formulas */ - private SheetRecordCollectingListener workbookBuildingListener; - - // excel2003Workbook(工作薄) - private HSSFWorkbook stubWorkbook; - - // Records we pick up as we process - private SSTRecord sstRecord; - - private FormatTrackingHSSFListener formatListener; - - // Table index(表索引) - private int sheetIndex = -1; - - private BoundSheetRecord[] orderedBSRs; - - @SuppressWarnings("unchecked") - private ArrayList boundSheetRecords = new ArrayList(); - - // For handling formulas with string results - private int nextRow; - - private int nextColumn; - - private boolean outputNextStringRecord; - - // Current line(当前行) - private int curRow = 0; - - // a container that stores row records(存储行记录的容器) - private List rowlist = new ArrayList(); - - @SuppressWarnings("unused") - private String sheetName; - - private IExcelRowDeal excelRowDeal; - - public void init(IExcelRowDeal excelRowDeal, InputStream inputStream) { - this.excelRowDeal = excelRowDeal; - this.inputStream = inputStream; - } - - /** - * Traverse all the sheets under excel 遍历excel下所有的sheet - * - * @throws IOException - */ - public void process() throws IOException { - this.fs = new POIFSFileSystem(this.inputStream); - MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this); - formatListener = new FormatTrackingHSSFListener(listener); - HSSFEventFactory factory = new HSSFEventFactory(); - HSSFRequest request = new HSSFRequest(); - if (outputFormulaValues) { - request.addListenerForAllRecords(formatListener); - } else { - workbookBuildingListener = new SheetRecordCollectingListener(formatListener); - request.addListenerForAllRecords(workbookBuildingListener); - } - factory.processWorkbookEvents(request, fs); - } - - /** HSSFListener listener method, processing Record HSSFListener 监听方法，处理 Record */ - @Override - @SuppressWarnings("unchecked") - public void processRecord(Record record) { - int thisRow = -1; - int thisColumn = -1; - String thisStr = null; - String value = null; - switch (record.getSid()) { - case BoundSheetRecord.sid: - boundSheetRecords.add(record); - break; - case BOFRecord.sid: - BOFRecord br = (BOFRecord) record; - if (br.getType() == BOFRecord.TYPE_WORKSHEET) { - // Create a child workbook if needed(如果有需要，则建立子工作薄) - if (workbookBuildingListener != null && stubWorkbook == null) { - stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook(); - } - - sheetIndex++; - if (orderedBSRs == null) { - orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords); - } - sheetName = orderedBSRs[sheetIndex].getSheetname(); - } - break; - - case SSTRecord.sid: - sstRecord = (SSTRecord) record; - break; - - case BlankRecord.sid: - BlankRecord brec = (BlankRecord) record; - thisRow = brec.getRow(); - thisColumn = brec.getColumn(); - thisStr = ""; - rowlist.add(thisColumn, thisStr); - break; - case BoolErrRecord.sid: // Cell is boolean(单元格为布尔类型) - BoolErrRecord berec = (BoolErrRecord) record; - thisRow = berec.getRow(); - thisColumn = berec.getColumn(); - thisStr = berec.getBooleanValue() + ""; - rowlist.add(thisColumn, thisStr); - break; - - case FormulaRecord.sid: // Cell is a formula type(单元格为公式类型) - FormulaRecord frec = (FormulaRecord) record; - thisRow = frec.getRow(); - thisColumn = frec.getColumn(); - if (outputFormulaValues) { - if (Double.isNaN(frec.getValue())) { - // Formula result is a string - // This is stored in the next record - outputNextStringRecord = true; - nextRow = frec.getRow(); - nextColumn = frec.getColumn(); - } else { - thisStr = formatListener.formatNumberDateCell(frec); - } - } else { - thisStr = - '"' - + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) - + '"'; - } - rowlist.add(thisColumn, thisStr); - break; - case StringRecord.sid: // a string of formulas in a cell(单元格中公式的字符串) - if (outputNextStringRecord) { - // String for formula - StringRecord srec = (StringRecord) record; - thisStr = srec.getString(); - thisRow = nextRow; - thisColumn = nextColumn; - outputNextStringRecord = false; - } - break; - case LabelRecord.sid: - LabelRecord lrec = (LabelRecord) record; - curRow = thisRow = lrec.getRow(); - thisColumn = lrec.getColumn(); - value = lrec.getValue().trim(); - value = value.equals("") ? " " : value; - this.rowlist.add(thisColumn, value); - break; - case LabelSSTRecord.sid: // Cell is a string type(单元格为字符串类型) - LabelSSTRecord lsrec = (LabelSSTRecord) record; - curRow = thisRow = lsrec.getRow(); - thisColumn = lsrec.getColumn(); - if (sstRecord == null) { - rowlist.add(thisColumn, " "); - } else { - value = sstRecord.getString(lsrec.getSSTIndex()).toString().trim(); - value = value.equals("") ? " " : value; - rowlist.add(thisColumn, value); - } - break; - case NumberRecord.sid: // Cell is a numeric type(单元格为数字类型) - NumberRecord numrec = (NumberRecord) record; - curRow = thisRow = numrec.getRow(); - thisColumn = numrec.getColumn(); - value = formatListener.formatNumberDateCell(numrec).trim(); - value = value.equals("") ? "0" : value; - // Add column values to the container(向容器加入列值) - rowlist.add(thisColumn, value); - break; - default: - break; - } - - // Encountered a new line of operations(遇到新行的操作)( - if (thisRow != -1 && thisRow != lastRowNumber) { - lastColumnNumber = -1; - } - - // Null operation(空值的操作) - if (record instanceof MissingCellDummyRecord) { - MissingCellDummyRecord mc = (MissingCellDummyRecord) record; - curRow = thisRow = mc.getRow(); - thisColumn = mc.getColumn(); - rowlist.add(thisColumn, " "); - } - - // Update row and column values(更新行和列的值) - if (thisRow > -1) lastRowNumber = thisRow; - if (thisColumn > -1) lastColumnNumber = thisColumn; - - // End of line operation(行结束时的操作) - if (record instanceof LastCellOfRowDummyRecord) { - if (minColumns > 0) { - // Column value is re-empted(列值重新置空) - if (lastColumnNumber == -1) { - lastColumnNumber = 0; - } - } - lastColumnNumber = -1; - - // At the end of each line, the dealRow() method(每行结束时， dealRow() 方法) - excelRowDeal.dealRow(orderedBSRs, sheetIndex, curRow, rowlist); - // Empty container(清空容器) - rowlist.clear(); - } - } - - public void close() { - try { - if (fs != null) { - fs.close(); - } - } catch (Exception e) { - logger.info("ExcelXlsReader fs closed failed", e); - } - - try { - if (inputStream != null) { - inputStream.close(); - } - } catch (IOException e) { - logger.info("ExcelXlsReader inputStream closed failed", e); - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/FirstRowDeal.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/FirstRowDeal.java deleted file mode 100644 index 6924a3ebb9..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/FirstRowDeal.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.excel; - -import org.apache.poi.hssf.record.BoundSheetRecord; - -import java.util.ArrayList; -import java.util.List; - -class FirstRowDeal implements IExcelRowDeal { - - private List sheetNames = new ArrayList<>(); - private List row; - - public List getSheetNames() { - return sheetNames; - } - - public void setSheetNames(List sheetNames) { - this.sheetNames = sheetNames; - } - - public List getRow() { - return row; - } - - public void setRow(List row) { - this.row = row; - } - - @Override - public void dealRow( - BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List rowlist) { - for (BoundSheetRecord record : orderedBSRs) { - sheetNames.add(record.getSheetname()); - } - row = rowlist; - throw new ExcelAnalysisException("Finished to deal first row"); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/RowToCsvDeal.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/RowToCsvDeal.java deleted file mode 100644 index 7deccfb92a..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/RowToCsvDeal.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.excel; - -import org.apache.poi.hssf.record.BoundSheetRecord; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.List; -import java.util.Map; - -class RowToCsvDeal implements IExcelRowDeal { - - private Map params; - private List sheetNames; - private OutputStream outputStream; - private Boolean hasHeader; - private Boolean fisrtRow = true; - - public void init(Boolean hasHeader, List sheetNames, OutputStream outputStream) { - this.hasHeader = hasHeader; - this.sheetNames = sheetNames; - this.outputStream = outputStream; - } - - @Override - public void dealRow( - BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List rowlist) { - String sheetName = orderedBSRs[sheetIndex].getSheetname(); - if (sheetNames == null || sheetNames.isEmpty() || sheetNames.contains(sheetName)) { - if (!(curRow == 0 && hasHeader)) { - try { - if (fisrtRow) { - fisrtRow = false; - } else { - outputStream.write("\n".getBytes()); - } - int len = rowlist.size(); - for (int i = 0; i < len; i++) { - outputStream.write(rowlist.get(i).replaceAll("\n|\t", " ").getBytes("utf-8")); - if (i < len - 1) { - outputStream.write("\t".getBytes()); - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageExcelWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageExcelWriter.java deleted file mode 100644 index 6b2a98c72b..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageExcelWriter.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.excel; - -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.domain.DataType; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; - -import org.apache.commons.io.IOUtils; -import org.apache.poi.ss.usermodel.*; -import org.apache.poi.xssf.streaming.SXSSFSheet; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; - -import java.io.*; -import java.math.BigDecimal; -import java.util.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class StorageExcelWriter extends ExcelFsWriter { - - private static Logger logger = LoggerFactory.getLogger(StorageExcelWriter.class); - - private String charset; - private String sheetName; - private String dateFormat; - private OutputStream outputStream; - private boolean autoFormat; - protected SXSSFWorkbook workBook; - protected SXSSFSheet sheet; - private DataFormat format; - protected DataType[] types; - protected int rowPoint; - protected int columnCounter; - protected Map styles = new HashMap<>(); - private boolean isFlush = true; - private ByteArrayOutputStream os = new ByteArrayOutputStream(); - private ByteArrayInputStream is; - - public StorageExcelWriter( - String charset, - String sheetName, - String dateFormat, - OutputStream outputStream, - boolean autoFormat) { - this.charset = charset; - this.sheetName = sheetName; - this.dateFormat = dateFormat; - this.outputStream = outputStream; - this.autoFormat = autoFormat; - } - - public void init() { - workBook = new SXSSFWorkbook(); - sheet = workBook.createSheet(sheetName); - } - - public CellStyle getDefaultHeadStyle() { - Font headerFont = workBook.createFont(); - headerFont.setBold(true); - headerFont.setFontHeightInPoints((short) 14); - headerFont.setColor(IndexedColors.RED.getIndex()); - CellStyle headerCellStyle = workBook.createCellStyle(); - headerCellStyle.setFont(headerFont); - return headerCellStyle; - } - - public Workbook getWorkBook() { - // 自适应列宽 - sheet.trackAllColumnsForAutoSizing(); - for (int elem = 0; elem <= columnCounter; elem++) { - sheet.autoSizeColumn(elem); - } - return workBook; - } - - public CellStyle createCellStyle(DataType dataType) { - CellStyle style = workBook.createCellStyle(); - format = workBook.createDataFormat(); - style.setDataFormat(format.getFormat("@")); - - if (autoFormat) { - switch (dataType) { - case StringType: - case CharType: - case VarcharType: - style.setDataFormat(format.getFormat("@")); - break; - case TinyIntType: - case ShortIntType: - case IntType: - style.setDataFormat(format.getFormat("#")); - break; - case LongType: - case BigIntType: - style.setDataFormat(format.getFormat("#.##E+00")); - break; - case FloatType: - style.setDataFormat(format.getFormat("#.0000000000")); - break; - case DoubleType: - style.setDataFormat(format.getFormat("#.0000000000")); - break; - case DateType: - case TimestampType: - style.setDataFormat(format.getFormat("m/d/yy h:mm")); - break; - case DecimalType: - case BigDecimalType: - style.setDataFormat(format.getFormat("#.000000000")); - break; - default: - style.setDataFormat(format.getFormat("@")); - } - } - return style; - } - - public CellStyle getCellStyle(DataType dataType) { - CellStyle style = styles.get(dataType.getTypeName()); - if (style == null) { - CellStyle newStyle = createCellStyle(dataType); - styles.put(dataType.getTypeName(), newStyle); - return newStyle; - } else { - return style; - } - } - - @Override - public void addMetaData(MetaData metaData) throws IOException { - init(); - Row tableHead = sheet.createRow(0); - Column[] columns = ((TableMetaData) metaData).getColumns(); - List columnType = new ArrayList<>(); - for (int i = 0; i < columns.length; i++) { - Cell headCell = tableHead.createCell(columnCounter); - headCell.setCellValue(columns[i].getColumnName()); - headCell.setCellStyle(getDefaultHeadStyle()); - columnType.add(columns[i].getDataType()); - columnCounter++; - } - types = columnType.toArray(new DataType[0]); - rowPoint++; - } - - @Override - public void addRecord(Record record) throws IOException { - // TODO: 是否需要替换null值 - Row tableBody = sheet.createRow(rowPoint); - int colunmPoint = 0; - Object[] excelRecord = ((TableRecord) record).row; - for (Object elem : excelRecord) { - Cell cell = tableBody.createCell(colunmPoint); - DataType dataType = types[colunmPoint]; - if (autoFormat) { - setCellTypeValue(dataType, elem, cell); - } else { - cell.setCellValue(DataType.valueToString(elem)); - } - cell.setCellStyle(getCellStyle(dataType)); - colunmPoint++; - } - rowPoint++; - } - - private void setCellTypeValue(DataType dataType, Object elem, Cell cell) { - if (null == elem) return; - - try { - switch (dataType) { - case StringType: - case CharType: - case VarcharType: - cell.setCellValue(DataType.valueToString(elem)); - break; - case TinyIntType: - case ShortIntType: - case IntType: - cell.setCellValue(Integer.valueOf(elem.toString())); - break; - case LongType: - case BigIntType: - cell.setCellValue(Long.valueOf(elem.toString())); - break; - case FloatType: - cell.setCellValue(Float.valueOf(elem.toString())); - break; - case DoubleType: - doubleCheck(elem.toString()); - cell.setCellValue(Double.valueOf(elem.toString())); - break; - case DateType: - case TimestampType: - cell.setCellValue(getDate(elem)); - break; - case DecimalType: - case BigDecimalType: - doubleCheck(DataType.valueToString(elem)); - cell.setCellValue(Double.valueOf(DataType.valueToString(elem))); - break; - default: - cell.setCellValue(DataType.valueToString(elem)); - } - } catch (Exception e) { - cell.setCellValue(DataType.valueToString(elem)); - } - } - - private Date getDate(Object value) { - if (value instanceof Date) { - return (Date) value; - } else { - throw new NumberFormatException( - "Value " - + value - + " with class : " - + value.getClass().getName() - + " is not a valid type of Date."); - } - } - - /** - * Check whether the double exceeds the number of digits, which will affect the data accuracy - * - * @param elemValue - */ - private void doubleCheck(String elemValue) { - BigDecimal value = new BigDecimal(elemValue).stripTrailingZeros(); - if ((value.precision() - value.scale()) > 15) { - throw new NumberFormatException( - "Value " + elemValue + " error : This data exceeds 15 significant digits."); - } - } - - @Override - public void flush() { - try { - getWorkBook().write(os); - } catch (IOException e) { - logger.warn("flush fail", e); - } - byte[] content = os.toByteArray(); - is = new ByteArrayInputStream(content); - byte[] buffer = new byte[1024]; - int bytesRead = 0; - while (isFlush) { - try { - bytesRead = is.read(buffer, 0, 1024); - if (bytesRead == -1) { - isFlush = false; - } else { - outputStream.write(buffer, 0, bytesRead); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - @Override - public void close() { - if (isFlush) { - flush(); - } - IOUtils.closeQuietly(outputStream); - IOUtils.closeQuietly(is); - IOUtils.closeQuietly(os); - IOUtils.closeQuietly(workBook); - } - - @Override - public String getCharset() { - return this.charset; - } - - @Override - public String getSheetName() { - return this.sheetName; - } - - @Override - public String getDateFormat() { - return this.dateFormat; - } - - @Override - public boolean isAutoFormat() { - return this.autoFormat; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java index 246fb79bc3..39d89c3d96 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java @@ -21,12 +21,17 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.openxml4j.util.ZipSecureFile; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +66,7 @@ public static String excelToCsv( throws Exception { String hdfsPath = "/tmp/" + StorageUtils.getJvmUser() + "/" + System.currentTimeMillis() + ".csv"; - LOG.info("The excel to csv with hdfsPath:" + hdfsPath); + LOG.info("The excel to csv with hdfs path:" + hdfsPath); ExcelXlsReader xlsReader = new ExcelXlsReader(); RowToCsvDeal rowToCsvDeal = new RowToCsvDeal(); OutputStream out = null; @@ -81,4 +86,44 @@ public static String excelToCsv( } return hdfsPath; } + + public static Map>> getSheetsInfo( + InputStream inputStream, Boolean hasHeader) { + // use xls file + Workbook workbook = null; + try { + // 压缩膨胀比率，处理excel行或者列过多的情况，不能设置再小了，会导致内存过大 + ZipSecureFile.setMinInflateRatio(0.005); + workbook = new HSSFWorkbook(inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + // 使用完最后需要还原 + ZipSecureFile.setMinInflateRatio(0.01); + } + Map>> res = new LinkedHashMap<>(workbook.getNumberOfSheets()); + // foreach Sheet + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + Sheet sheet = workbook.getSheetAt(i); + + List> rowList = new ArrayList<>(); + + // get first row as column name + Row headerRow = sheet.getRow(0); + + // foreach column + for (int j = 0; j < headerRow.getPhysicalNumberOfCells(); j++) { + Map sheetMap = new LinkedHashMap<>(); + Cell cell = headerRow.getCell(j); + if (hasHeader) { + sheetMap.put(cell.getStringCellValue(), "string"); + } else { + sheetMap.put("col_" + (j + 1), "string"); + } + rowList.add(sheetMap); + } + res.put(sheet.getSheetName(), rowList); + } + return res; + } } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java index 7cbf579a9d..c2418f4c33 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsxUtils.java @@ -23,10 +23,9 @@ import org.apache.poi.ss.usermodel.Workbook; import java.io.File; +import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.*; import com.github.pjfanning.xlsx.StreamingReader; @@ -79,4 +78,54 @@ public static List> getBasicInfo(InputStream inputStream, File file } } } + + public static Map>> getAllSheetInfo( + InputStream inputStream, File file, Boolean hasHeader) throws IOException { + try { + Workbook wb = null; + if (inputStream != null) { + wb = + StreamingReader.builder() + // number of rows to keep in memory (defaults to 10) + .rowCacheSize(2) + .open(inputStream); + } else { + wb = + StreamingReader.builder() + // number of rows to keep in memory (defaults to 10) + .rowCacheSize(2) + .open(file); + } + Map>> res = new LinkedHashMap<>(wb.getNumberOfSheets()); + for (Sheet sheet : wb) { + Iterator iterator = sheet.iterator(); + Row row = null; + while (iterator.hasNext() && row == null) { + row = iterator.next(); + } + List> rowList = new ArrayList<>(); + if (row == null) { + res.put(sheet.getSheetName(), rowList); + continue; + } + int cellIdx = 0; + for (Cell cell : row) { + Map item = new LinkedHashMap<>(); + if (hasHeader) { + item.put(cell.getStringCellValue(), "string"); + } else { + item.put("col_" + (cellIdx + 1), "string"); + } + cellIdx++; + rowList.add(item); + } + res.put(sheet.getSheetName(), rowList); + } + return res; + } finally { + if (inputStream != null) { + inputStream.close(); + } + } + } } diff --git a/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/ExecuteException.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColLengthExceedException.java similarity index 65% rename from linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/ExecuteException.java rename to linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColLengthExceedException.java index dac32f0984..955a8e1d60 100644 --- a/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/ExecuteException.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColLengthExceedException.java @@ -15,12 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.python.exception; +package org.apache.linkis.storage.exception; -import org.apache.linkis.common.exception.ErrorException; +public class ColLengthExceedException extends StorageWarnException { -public class ExecuteException extends ErrorException { - public ExecuteException(int errCode, String desc) { + public ColLengthExceedException(int errCode, String desc) { super(errCode, desc); } + + public ColLengthExceedException( + int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + + public ColLengthExceedException(int errCode, String desc, Throwable t) { + super(errCode, desc, t); + } } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColumnIndexExceedException.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColumnIndexExceedException.java new file mode 100644 index 0000000000..969b19d20b --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/ColumnIndexExceedException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.exception; + +public class ColumnIndexExceedException extends StorageWarnException { + + public ColumnIndexExceedException(int errCode, String desc) { + super(errCode, desc); + } + + public ColumnIndexExceedException( + int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + + public ColumnIndexExceedException(int errCode, String desc, Throwable t) { + super(errCode, desc, t); + } +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java index 7a8452141b..ae66c1cf99 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java @@ -21,10 +21,8 @@ public enum StorageErrorCode { /** */ FS_NOT_INIT(53001, "please init first"), - INCONSISTENT_DATA(53001, "Inconsistent row data read,read %s,need rowLen %s"), FS_OOM(53002, "OOM occurred while reading the file"), - FS_ERROR(53003, "Failed to operation fs"), READ_PARQUET_FAILED(53004, "Failed to read parquet file"), diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java index 8103c6f3de..9f53a6249b 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildHDFSFileSystem.java @@ -21,7 +21,7 @@ import org.apache.linkis.storage.factory.BuildFactory; import org.apache.linkis.storage.fs.FileSystem; import org.apache.linkis.storage.fs.impl.HDFSFileSystem; -import org.apache.linkis.storage.io.IOMethodInterceptorFactory; +import org.apache.linkis.storage.io.IOMethodInterceptorCreator$; import org.apache.linkis.storage.utils.StorageUtils; import org.springframework.cglib.proxy.Enhancer; @@ -46,7 +46,7 @@ public Fs getFs(String user, String proxyUser) { // TODO Agent user(代理的用户) Enhancer enhancer = new Enhancer(); enhancer.setSuperclass(HDFSFileSystem.class.getSuperclass()); - enhancer.setCallback(IOMethodInterceptorFactory.getIOMethodInterceptor(fsName())); + enhancer.setCallback(IOMethodInterceptorCreator$.MODULE$.getIOMethodInterceptor(fsName())); fs = (FileSystem) enhancer.create(); } fs.setUser(proxyUser); @@ -63,6 +63,6 @@ public Fs getFs(String user, String proxyUser, String label) { @Override public String fsName() { - return StorageUtils.HDFS; + return StorageUtils.HDFS(); } } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java index bcd61c5735..ef88cec36d 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildLocalFileSystem.java @@ -21,7 +21,7 @@ import org.apache.linkis.storage.factory.BuildFactory; import org.apache.linkis.storage.fs.FileSystem; import org.apache.linkis.storage.fs.impl.LocalFileSystem; -import org.apache.linkis.storage.io.IOMethodInterceptorFactory; +import org.apache.linkis.storage.io.IOMethodInterceptorCreator$; import org.apache.linkis.storage.utils.StorageConfiguration; import org.apache.linkis.storage.utils.StorageUtils; @@ -34,7 +34,7 @@ public Fs getFs(String user, String proxyUser) { FileSystem fs = null; if (StorageUtils.isIOProxy()) { if (user.equals(proxyUser)) { - if ((Boolean) StorageConfiguration.IS_SHARE_NODE.getValue()) { + if ((Boolean) StorageConfiguration.IS_SHARE_NODE().getValue()) { fs = new LocalFileSystem(); } else { fs = getProxyFs(); @@ -58,12 +58,12 @@ public Fs getFs(String user, String proxyUser, String label) { private FileSystem getProxyFs() { Enhancer enhancer = new Enhancer(); enhancer.setSuperclass(LocalFileSystem.class.getSuperclass()); - enhancer.setCallback(IOMethodInterceptorFactory.getIOMethodInterceptor(fsName())); + enhancer.setCallback(IOMethodInterceptorCreator$.MODULE$.getIOMethodInterceptor(fsName())); return (FileSystem) enhancer.create(); } @Override public String fsName() { - return StorageUtils.FILE; + return StorageUtils.FILE(); } } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildOSSSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildOSSSystem.java deleted file mode 100644 index ba1bd7abef..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildOSSSystem.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.factory.impl; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.storage.factory.BuildFactory; -import org.apache.linkis.storage.fs.impl.OSSFileSystem; -import org.apache.linkis.storage.utils.StorageUtils; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BuildOSSSystem implements BuildFactory { - - private static final Logger LOG = LoggerFactory.getLogger(BuildOSSSystem.class); - - /** - * get file system - * - * @param user - * @param proxyUser - * @return - */ - @Override - public Fs getFs(String user, String proxyUser) { - OSSFileSystem fs = new OSSFileSystem(); - try { - fs.init(null); - } catch (IOException e) { - LOG.warn("get file system failed", e); - } - fs.setUser(user); - return fs; - } - - @Override - public Fs getFs(String user, String proxyUser, String label) { - OSSFileSystem fs = new OSSFileSystem(); - try { - fs.init(null); - } catch (IOException e) { - LOG.warn("get file system failed", e); - } - fs.setUser(proxyUser); - fs.setLabel(label); - return fs; - } - - @Override - public String fsName() { - return StorageUtils.OSS; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildS3FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildS3FileSystem.java deleted file mode 100644 index 44082e5898..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/factory/impl/BuildS3FileSystem.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.factory.impl; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.storage.factory.BuildFactory; -import org.apache.linkis.storage.fs.impl.S3FileSystem; -import org.apache.linkis.storage.utils.StorageUtils; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BuildS3FileSystem implements BuildFactory { - private static final Logger LOG = LoggerFactory.getLogger(BuildS3FileSystem.class); - - @Override - public Fs getFs(String user, String proxyUser) { - S3FileSystem fs = new S3FileSystem(); - try { - fs.init(null); - } catch (IOException e) { - LOG.warn("get file system failed", e); - } - fs.setUser(user); - return fs; - } - - @Override - public Fs getFs(String user, String proxyUser, String label) { - S3FileSystem fs = new S3FileSystem(); - try { - fs.init(null); - } catch (IOException e) { - LOG.warn("get file system failed", e); - } - fs.setUser(user); - fs.setLabel(label); - return fs; - } - - @Override - public String fsName() { - return StorageUtils.S3; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java index 3067383b6c..ac828267bf 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java @@ -51,6 +51,10 @@ public String getDefaultFolderPerm() { public abstract long getUsableSpace(FsPath dest) throws IOException; + public abstract long getLength(FsPath dest) throws IOException; + + public abstract String checkSum(FsPath dest) throws IOException; + public abstract boolean canExecute(FsPath dest) throws IOException; public abstract boolean setOwner(FsPath dest, String user, String group) throws IOException; @@ -99,7 +103,7 @@ protected FsPath getParentPath(String path) { } else { parentPath = path.substring(0, path.lastIndexOf("/")); } - LOG.info("Get Parent Path:" + parentPath); + LOG.info("Get parent path:" + parentPath); return new FsPath(parentPath); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java index f40d75c040..c4f4814149 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java @@ -31,9 +31,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; @@ -82,7 +80,7 @@ public long getUsableSpace(FsPath dest) throws IOException { @Override public boolean canExecute(FsPath dest) throws IOException { - return canAccess(dest, FsAction.EXECUTE); + return canAccess(dest, FsAction.EXECUTE, this.user); } @Override @@ -162,7 +160,8 @@ public FsPathListWithError listPathWithError(FsPath path) throws IOException { List fsPaths = new ArrayList(); for (FileStatus f : stat) { fsPaths.add( - fillStorageFile(new FsPath(StorageUtils.HDFS_SCHEMA + f.getPath().toUri().getPath()), f)); + fillStorageFile( + new FsPath(StorageUtils.HDFS_SCHEMA() + f.getPath().toUri().getPath()), f)); } if (fsPaths.isEmpty()) { return null; @@ -174,35 +173,39 @@ public FsPathListWithError listPathWithError(FsPath path) throws IOException { @Override public void init(Map properties) throws IOException { if (MapUtils.isNotEmpty(properties) - && properties.containsKey(StorageConfiguration.PROXY_USER.key())) { - user = StorageConfiguration.PROXY_USER.getValue(properties); + && properties.containsKey(StorageConfiguration.PROXY_USER().key())) { + user = StorageConfiguration.PROXY_USER().getValue(properties); + properties.remove(StorageConfiguration.PROXY_USER().key()); } if (user == null) { throw new IOException("User cannot be empty(用户不能为空)"); } - - if (label == null && (boolean) Configuration.IS_MULTIPLE_YARN_CLUSTER()) { - label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL.getValue(); + if (label == null && Configuration.IS_MULTIPLE_YARN_CLUSTER()) { + label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL().getValue(); } - conf = HDFSUtils.getConfigurationByLabel(user, label); - + /** if properties is null do not to create conf */ if (MapUtils.isNotEmpty(properties)) { - for (String key : properties.keySet()) { - String v = properties.get(key); - if (StringUtils.isNotEmpty(v)) { - conf.set(key, v); + conf = HDFSUtils.getConfigurationByLabel(user, label); + if (MapUtils.isNotEmpty(properties)) { + for (String key : properties.keySet()) { + String v = properties.get(key); + if (StringUtils.isNotEmpty(v)) { + conf.set(key, v); + } } } } - if (StorageConfiguration.FS_CACHE_DISABLE.getValue()) { - conf.set("fs.hdfs.impl.disable.cache", "true"); + if (null != conf) { + fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf); + } else { + fs = HDFSUtils.getHDFSUserFileSystem(user, label); } - fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf); + if (fs == null) { throw new IOException("init HDFS FileSystem failed!"); } - if (StorageConfiguration.FS_CHECKSUM_DISBALE.getValue()) { + if (StorageConfiguration.FS_CHECKSUM_DISBALE().getValue()) { fs.setVerifyChecksum(false); fs.setWriteChecksum(false); } @@ -215,7 +218,7 @@ public String fsName() { @Override public String rootUserName() { - return StorageConfiguration.HDFS_ROOT_USER.getValue(); + return StorageConfiguration.HDFS_ROOT_USER().getValue(); } @Override @@ -248,7 +251,6 @@ public OutputStream write(FsPath dest, boolean overwrite) throws IOException { return fs.append(new Path(path)); } else { OutputStream out = fs.create(new Path(path), true); - this.setPermission(dest, this.getDefaultFilePerm()); return out; } } @@ -293,17 +295,16 @@ public List list(FsPath path) throws IOException { @Override public boolean canRead(FsPath dest) throws IOException { - return canAccess(dest, FsAction.READ); + return canAccess(dest, FsAction.READ, this.user); } - @Override - public boolean canRead(FsPath fsPath, String s) throws IOException { - return false; + public boolean canRead(FsPath dest, String user) throws IOException { + return canAccess(dest, FsAction.READ, user); } @Override public boolean canWrite(FsPath dest) throws IOException { - return canAccess(dest, FsAction.WRITE); + return canAccess(dest, FsAction.WRITE, this.user); } @Override @@ -313,10 +314,10 @@ public boolean exists(FsPath dest) throws IOException { } catch (IOException e) { String message = e.getMessage(); String rootCauseMessage = ExceptionUtils.getRootCauseMessage(e); - if ((message != null && message.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS)) + if ((message != null && message.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS())) || (rootCauseMessage != null - && rootCauseMessage.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS))) { - logger.info("Failed to execute exists, retry", e); + && rootCauseMessage.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS()))) { + logger.info("Failed to execute exists for user {}, retry", user, e); resetRootHdfs(); return fs.exists(new Path(checkHDFSPath(dest.getPath()))); } else { @@ -334,8 +335,12 @@ private void resetRootHdfs() { } else { HDFSUtils.closeHDFSFIleSystem(fs, user, label); } - logger.warn(user + "FS reset close."); - fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf); + logger.warn("{} FS reset close.", user); + if (null != conf) { + fs = HDFSUtils.getHDFSUserFileSystem(user, label, conf); + } else { + fs = HDFSUtils.getHDFSUserFileSystem(user, label); + } } } } @@ -362,7 +367,7 @@ public boolean renameTo(FsPath oldDest, FsPath newDest) throws IOException { @Override public void close() throws IOException { if (null != fs) { - HDFSUtils.closeHDFSFIleSystem(fs, user, label); + HDFSUtils.closeHDFSFIleSystem(fs, user); } else { logger.warn("FS was null, cannot close."); } @@ -388,7 +393,7 @@ private FsPath fillStorageFile(FsPath fsPath, FileStatus fileStatus) throws IOEx return fsPath; } - private boolean canAccess(FsPath fsPath, FsAction access) throws IOException { + private boolean canAccess(FsPath fsPath, FsAction access, String user) throws IOException { String path = checkHDFSPath(fsPath.getPath()); if (!exists(fsPath)) { throw new IOException("directory or file not exists: " + path); @@ -396,12 +401,12 @@ private boolean canAccess(FsPath fsPath, FsAction access) throws IOException { FileStatus f = fs.getFileStatus(new Path(path)); FsPermission permission = f.getPermission(); - UserGroupInformation ugi = HDFSUtils.getUserGroupInformation(user, label); + UserGroupInformation ugi = HDFSUtils.getUserGroupInformation(user); String[] groupNames; try { groupNames = ugi.getGroupNames(); } catch (NullPointerException e) { - if ((Boolean) Configuration.IS_TEST_MODE().getValue()) { + if ((Boolean) org.apache.linkis.common.conf.Configuration.IS_TEST_MODE().getValue()) { groupNames = new String[] {"hadoop"}; } else { throw e; @@ -433,9 +438,9 @@ public void setLabel(String label) { private String checkHDFSPath(String path) { try { - boolean checkHdfsPath = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_CHECK_ON.getValue(); + boolean checkHdfsPath = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_CHECK_ON().getValue(); if (checkHdfsPath) { - boolean rmHdfsPrefix = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_REMOVE.getValue(); + boolean rmHdfsPrefix = (boolean) StorageConfiguration.HDFS_PATH_PREFIX_REMOVE().getValue(); if (rmHdfsPrefix) { if (StringUtils.isBlank(path)) { return path; @@ -471,4 +476,21 @@ private String checkHDFSPath(String path) { } return path; } + + @Override + public long getLength(FsPath dest) throws IOException { + FileStatus fileStatus = fs.getFileStatus(new Path(checkHDFSPath(dest.getPath()))); + return fileStatus.getLen(); + } + + @Override + public String checkSum(FsPath dest) throws IOException { + String path = checkHDFSPath(dest.getPath()); + if (!exists(dest)) { + throw new IOException("directory or file not exists: " + path); + } + MD5MD5CRC32FileChecksum fileChecksum = + (MD5MD5CRC32FileChecksum) fs.getFileChecksum(new Path(path)); + return fileChecksum.toString().split(":")[1]; + } } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java index 9516e90361..a03a25950e 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java @@ -47,12 +47,7 @@ import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Stack; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,7 +108,7 @@ public boolean setOwner(FsPath dest, String user, String group) throws IOExcepti if (group != null) { setGroup(dest, group); } - setGroup(dest, StorageConfiguration.STORAGE_USER_GROUP.getValue()); + setGroup(dest, StorageConfiguration.STORAGE_USER_GROUP().getValue()); return true; } @@ -136,7 +131,7 @@ public boolean setOwner(FsPath dest, String user) throws IOException { @Override public boolean setGroup(FsPath dest, String group) throws IOException { - LOG.info("Set group with path:" + dest.getPath() + "and group:" + user); + LOG.info("Set group with path:" + dest.getPath() + " and group:" + group); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setGroup skip"); return true; @@ -262,9 +257,17 @@ public FsPathListWithError listPathWithError(FsPath path) throws IOException { LOG.info("Try to list path:" + path.getPath() + " with error msg"); if (files != null) { List rtn = new ArrayList(); + Set fileNameSet = new HashSet<>(); + fileNameSet.add(path.getPath().trim()); String message = ""; for (File f : files) { try { + if (fileNameSet.contains(f.getPath())) { + LOG.info("File {} is duplicate", f.getPath()); + continue; + } else { + fileNameSet.add(f.getParent().trim()); + } rtn.add(get(f.getPath())); } catch (Throwable e) { LOG.warn("Failed to list path:", e); @@ -289,15 +292,15 @@ public void init(Map properties) throws IOException { if (MapUtils.isNotEmpty(properties)) { this.properties = properties; - if (properties.containsKey(StorageConfiguration.PROXY_USER.key())) { - user = StorageConfiguration.PROXY_USER.getValue(properties); + if (properties.containsKey(StorageConfiguration.PROXY_USER().key())) { + user = StorageConfiguration.PROXY_USER().getValue(properties); } - group = StorageConfiguration.STORAGE_USER_GROUP.getValue(properties); + group = StorageConfiguration.STORAGE_USER_GROUP().getValue(properties); } else { this.properties = new HashMap(); } if (FsPath.WINDOWS) { - group = StorageConfiguration.STORAGE_USER_GROUP.getValue(properties); + group = StorageConfiguration.STORAGE_USER_GROUP().getValue(properties); } if (StringUtils.isEmpty(group)) { String groupInfo; @@ -320,7 +323,7 @@ public String fsName() { @Override public String rootUserName() { - return StorageConfiguration.LOCAL_ROOT_USER.getValue(); + return StorageConfiguration.LOCAL_ROOT_USER().getValue(); } @Override @@ -423,7 +426,7 @@ public boolean canRead(FsPath dest) throws IOException { } @Override - public boolean canRead(FsPath fsPath, String s) throws IOException { + public boolean canRead(FsPath dest, String user) throws IOException { return false; } @@ -494,4 +497,16 @@ private String getOwner(String path) throws IOException { PosixFileAttributes attr = Files.readAttributes(Paths.get(path), PosixFileAttributes.class); return attr.owner().getName(); } + + @Override + public long getLength(FsPath dest) throws IOException { + String path = dest.getPath(); + LOG.info("Get file length with path:" + path); + return new File(path).length(); + } + + @Override + public String checkSum(FsPath dest) { + return null; + } } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java deleted file mode 100644 index 2d52b83049..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.fs.impl; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.hadoop.common.utils.HDFSUtils; -import org.apache.linkis.storage.conf.LinkisStorageConf; -import org.apache.linkis.storage.domain.FsPathListWithError; -import org.apache.linkis.storage.fs.FileSystem; -import org.apache.linkis.storage.utils.StorageConfiguration; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class OSSFileSystem extends FileSystem { - - public static final String OSS_PREFIX = "oss://"; - private org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem fs = null; - private Configuration conf = null; - - private String label = null; - - private static final Logger logger = LoggerFactory.getLogger(OSSFileSystem.class); - - /** File System abstract method start */ - @Override - public String listRoot() throws IOException { - return "/"; - } - - @Override - public long getTotalSpace(FsPath dest) throws IOException { - return 0; - } - - @Override - public long getFreeSpace(FsPath dest) throws IOException { - return 0; - } - - @Override - public long getUsableSpace(FsPath dest) throws IOException { - return 0; - } - - @Override - public boolean canExecute(FsPath dest) throws IOException { - return true; - } - - @Override - public boolean setOwner(FsPath dest, String user, String group) throws IOException { - return true; - } - - @Override - public boolean setOwner(FsPath dest, String user) throws IOException { - return true; - } - - @Override - public boolean setGroup(FsPath dest, String group) throws IOException { - return true; - } - - @Override - public boolean mkdir(FsPath dest) throws IOException { - String path = checkOSSPath(dest.getPath()); - if (!canExecute(getParentPath(path))) { - throw new IOException("You have not permission to access path " + path); - } - boolean result = - fs.mkdirs(new Path(path), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); - this.setPermission(new FsPath(path), this.getDefaultFolderPerm()); - return result; - } - - @Override - public boolean mkdirs(FsPath dest) throws IOException { - String path = checkOSSPath(dest.getPath()); - FsPath parentPath = getParentPath(path); - while (!exists(parentPath)) { - parentPath = getParentPath(parentPath.getPath()); - } - return fs.mkdirs(new Path(path), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); - } - - @Override - public boolean setPermission(FsPath dest, String permission) throws IOException { - return true; - } - - @Override - public FsPathListWithError listPathWithError(FsPath path) throws IOException { - FileStatus[] stat = fs.listStatus(new Path(checkOSSPath(path.getPath()))); - List fsPaths = new ArrayList(); - for (FileStatus f : stat) { - fsPaths.add( - fillStorageFile( - new FsPath( - StorageUtils.OSS_SCHEMA - + StorageConfiguration.OSS_ACCESS_BUCKET_NAME.getValue() - + "/" - + f.getPath().toUri().getPath()), - f)); - } - if (fsPaths.isEmpty()) { - return null; - } - return new FsPathListWithError(fsPaths, ""); - } - - /** FS interface method start */ - @Override - public void init(Map properties) throws IOException { - // read origin configs from hadoop conf - if (label == null - && (boolean) org.apache.linkis.common.conf.Configuration.IS_MULTIPLE_YARN_CLUSTER()) { - label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL.getValue(); - } - conf = HDFSUtils.getConfigurationByLabel(user, label); - - // origin configs - Map originProperties = Maps.newHashMap(); - originProperties.put("fs.oss.endpoint", StorageConfiguration.OSS_ENDPOINT.getValue()); - originProperties.put("fs.oss.accessKeyId", StorageConfiguration.OSS_ACCESS_KEY_ID.getValue()); - originProperties.put( - "fs.oss.accessKeySecret", StorageConfiguration.OSS_ACCESS_KEY_SECRET.getValue()); - for (String key : originProperties.keySet()) { - String value = originProperties.get(key); - if (StringUtils.isNotBlank(value)) { - conf.set(key, value); - } - } - - // additional configs - if (MapUtils.isNotEmpty(properties)) { - for (String key : properties.keySet()) { - String v = properties.get(key); - if (StringUtils.isNotBlank(v)) { - conf.set(key, v); - } - } - } - fs = new AliyunOSSFileSystem(); - try { - fs.initialize( - new URI(StorageUtils.OSS_SCHEMA + StorageConfiguration.OSS_ACCESS_BUCKET_NAME.getValue()), - conf); - } catch (URISyntaxException e) { - throw new IOException("init OSS FileSystem failed!"); - } - if (fs == null) { - throw new IOException("init OSS FileSystem failed!"); - } - } - - @Override - public String fsName() { - return StorageUtils.OSS; - } - - @Override - public String rootUserName() { - return null; - } - - @Override - public FsPath get(String dest) throws IOException { - String realPath = checkOSSPath(dest); - return fillStorageFile(new FsPath(realPath), fs.getFileStatus(new Path(realPath))); - } - - @Override - public InputStream read(FsPath dest) throws IOException { - if (!canRead(dest)) { - throw new IOException("You have not permission to access path " + dest.getPath()); - } - return fs.open(new Path(dest.getPath()), 128); - } - - @Override - public OutputStream write(FsPath dest, boolean overwrite) throws IOException { - String path = checkOSSPath(dest.getPath()); - if (!exists(dest)) { - if (!canWrite(dest.getParent())) { - throw new IOException("You have not permission to access path " + dest.getParent()); - } - } else { - if (!canWrite(dest)) { - throw new IOException("You have not permission to access path " + path); - } - } - OutputStream out = - fs.create( - new Path(path), - new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL), - overwrite, - 0, - (short) 0, - 0L, - null); - this.setPermission(dest, this.getDefaultFilePerm()); - return out; - } - - @Override - public boolean create(String dest) throws IOException { - if (!canExecute(getParentPath(dest))) { - throw new IOException("You have not permission to access path " + dest); - } - // to do - boolean result = fs.createNewFile(new Path(checkOSSPath(dest))); - this.setPermission(new FsPath(dest), this.getDefaultFilePerm()); - return result; - } - - @Override - public boolean copy(String origin, String dest) throws IOException { - if (!canExecute(getParentPath(dest))) { - throw new IOException("You have not permission to access path " + dest); - } - boolean res = - FileUtil.copy( - fs, - new Path(checkOSSPath(origin)), - fs, - new Path(checkOSSPath(dest)), - false, - true, - fs.getConf()); - this.setPermission(new FsPath(dest), this.getDefaultFilePerm()); - return res; - } - - @Override - public List list(FsPath path) throws IOException { - FileStatus[] stat = fs.listStatus(new Path(checkOSSPath(path.getPath()))); - List fsPaths = new ArrayList(); - for (FileStatus f : stat) { - fsPaths.add(fillStorageFile(new FsPath(f.getPath().toUri().toString()), f)); - } - return fsPaths; - } - - @Override - public boolean canRead(FsPath dest) throws IOException { - return true; - } - - @Override - public boolean canRead(FsPath fsPath, String s) throws IOException { - return false; - } - - @Override - public boolean canWrite(FsPath dest) throws IOException { - return true; - } - - @Override - public boolean exists(FsPath dest) throws IOException { - try { - return fs.exists(new Path(checkOSSPath(dest.getPath()))); - } catch (IOException e) { - String message = e.getMessage(); - String rootCauseMessage = ExceptionUtils.getRootCauseMessage(e); - if ((message != null && message.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS)) - || (rootCauseMessage != null - && rootCauseMessage.matches(LinkisStorageConf.HDFS_FILE_SYSTEM_REST_ERRS))) { - logger.info("Failed to execute exists, retry", e); - resetRootOSS(); - return fs.exists(new Path(checkOSSPath(dest.getPath()))); - } else { - throw e; - } - } - } - - private void resetRootOSS() throws IOException { - if (fs != null) { - synchronized (this) { - if (fs != null) { - fs.close(); - logger.warn(user + " FS reset close."); - init(null); - } - } - } - } - - @Override - public boolean delete(FsPath dest) throws IOException { - String path = checkOSSPath(dest.getPath()); - return fs.delete(new Path(path), true); - } - - @Override - public boolean renameTo(FsPath oldDest, FsPath newDest) throws IOException { - return fs.rename( - new Path(checkOSSPath(oldDest.getPath())), new Path(checkOSSPath(newDest.getPath()))); - } - - @Override - public void close() throws IOException { - if (null != fs) { - fs.close(); - } else { - logger.warn("FS was null, cannot close."); - } - } - - /** Utils method start */ - private FsPath fillStorageFile(FsPath fsPath, FileStatus fileStatus) throws IOException { - fsPath.setAccess_time(fileStatus.getAccessTime()); - fsPath.setModification_time(fileStatus.getModificationTime()); - fsPath.setOwner(fileStatus.getOwner()); - fsPath.setGroup(fileStatus.getGroup()); - fsPath.setIsdir(fileStatus.isDirectory()); - return fsPath; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - private static String checkOSSPath(String path) { - try { - boolean checkOSSPath = (boolean) StorageConfiguration.OSS_PATH_PREFIX_CHECK_ON.getValue(); - if (checkOSSPath) { - boolean rmOSSPrefix = (boolean) StorageConfiguration.OSS_PATH_PREFIX_REMOVE.getValue(); - if (rmOSSPrefix) { - if (StringUtils.isBlank(path)) { - return path; - } - if (path.startsWith(OSS_PREFIX)) { - int remainIndex = OSS_PREFIX.length(); - String[] t1 = path.substring(remainIndex).split("/", 2); - if (t1.length != 2) { - logger.warn("checkOSSPath Invalid path: " + path); - return path; - } - if (logger.isDebugEnabled()) { - logger.debug("checkOSSPath ori path : {}, after path : {}", path, "/" + t1[1]); - } - return "/" + t1[1]; - } else { - return path; - } - } - } - } catch (Exception e) { - logger.warn("checkOSSPath error. msg : " + e.getMessage() + " ", e); - } - return path; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java deleted file mode 100644 index 1e7a90532c..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.fs.impl; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.storage.domain.FsPathListWithError; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.fs.FileSystem; -import org.apache.linkis.storage.utils.StorageConfiguration; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.TO_BE_UNKNOW; - -public class S3FileSystem extends FileSystem { - private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); - private String accessKey; - private String secretKey; - - private String endPoint; - - private String region; - - private String bucket; - - private String label; - - private AmazonS3 s3Client; - - private static final String INIT_FILE_NAME = ".s3_dir_init"; - - @Override - public void init(Map properties) throws IOException { - accessKey = StorageConfiguration.S3_ACCESS_KEY.getValue(properties); - secretKey = StorageConfiguration.S3_SECRET_KEY.getValue(properties); - endPoint = StorageConfiguration.S3_ENDPOINT.getValue(properties); - bucket = StorageConfiguration.S3_BUCKET.getValue(properties); - region = StorageConfiguration.S3_REGION.getValue(properties); - - AwsClientBuilder.EndpointConfiguration endpointConfiguration = - new AwsClientBuilder.EndpointConfiguration(endPoint, region); - - BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey); - - AWSStaticCredentialsProvider StaticCredentials = - new AWSStaticCredentialsProvider(basicAWSCredentials); - - s3Client = - AmazonS3ClientBuilder.standard() - .withEndpointConfiguration(endpointConfiguration) - .withPathStyleAccessEnabled(true) - .withCredentials(StaticCredentials) - .build(); - } - - @Override - public String fsName() { - return StorageUtils.S3; - } - - @Override - public String rootUserName() { - return null; - } - - @Override - public FsPath get(String dest) throws IOException { - FsPath ret = new FsPath(dest); - if (exists(ret)) { - return ret; - } else { - logger.warn("File or folder does not exist or file name is garbled(文件或者文件夹不存在或者文件名乱码)"); - throw new StorageWarnException( - TO_BE_UNKNOW.getErrorCode(), - "File or folder does not exist or file name is garbled(文件或者文件夹不存在或者文件名乱码)"); - } - } - - @Override - public InputStream read(FsPath dest) throws IOException { - try { - return s3Client.getObject(bucket, buildPrefix(dest.getPath(), false)).getObjectContent(); - } catch (AmazonS3Exception e) { - throw new IOException("You have not permission to access path " + dest.getPath()); - } - } - - @Override - public OutputStream write(FsPath dest, boolean overwrite) throws IOException { - try (InputStream inputStream = read(dest); - OutputStream outputStream = - new S3OutputStream(s3Client, bucket, buildPrefix(dest.getPath(), false))) { - if (!overwrite) { - IOUtils.copy(inputStream, outputStream); - } - return outputStream; - } - } - - @Override - public boolean create(String dest) throws IOException { - if (exists(new FsPath(dest))) { - return false; - } - s3Client.putObject(bucket, dest, ""); - return true; - } - - @Override - public List list(FsPath path) throws IOException { - try { - if (!StringUtils.isEmpty(path.getPath())) { - ListObjectsV2Result listObjectsV2Result = s3Client.listObjectsV2(bucket, path.getPath()); - List s3ObjectSummaries = listObjectsV2Result.getObjectSummaries(); - return s3ObjectSummaries.stream() - .filter(summary -> !isInitFile(summary)) - .map( - summary -> { - FsPath newPath = new FsPath(buildPath(summary.getKey())); - return fillStorageFile(newPath, summary); - }) - .collect(Collectors.toList()); - } - } catch (AmazonS3Exception e) { - throw new IOException("You have not permission to access path " + path.getPath()); - } - - return new ArrayList<>(); - } - - @Override - public FsPathListWithError listPathWithError(FsPath path) throws IOException { - return listPathWithError(path, true); - } - - public FsPathListWithError listPathWithError(FsPath path, boolean ignoreInitFile) - throws IOException { - List rtn = new ArrayList<>(); - try { - if (!StringUtils.isEmpty(path.getPath())) { - ListObjectsV2Request listObjectsV2Request = - new ListObjectsV2Request() - .withBucketName(bucket) - .withPrefix(buildPrefix(path.getPath())) - .withDelimiter("/"); - ListObjectsV2Result dirResult = s3Client.listObjectsV2(listObjectsV2Request); - List s3ObjectSummaries = dirResult.getObjectSummaries(); - List commonPrefixes = dirResult.getCommonPrefixes(); - if (s3ObjectSummaries != null) { - for (S3ObjectSummary summary : s3ObjectSummaries) { - if (isInitFile(summary) && ignoreInitFile) continue; - FsPath newPath = new FsPath(buildPath(summary.getKey())); - rtn.add(fillStorageFile(newPath, summary)); - } - } - if (commonPrefixes != null) { - for (String dir : commonPrefixes) { - FsPath newPath = new FsPath(buildPath(dir)); - newPath.setIsdir(true); - rtn.add(newPath); - } - } - return new FsPathListWithError(rtn, ""); - } - } catch (AmazonS3Exception e) { - throw new IOException("You have not permission to access path " + path.getPath()); - } - - return null; - } - - @Override - public boolean exists(FsPath dest) throws IOException { - try { - if (new File(dest.getPath()).getName().contains(".")) { - return existsFile(dest); - } - ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); - listObjectsV2Request - .withBucketName(bucket) - .withPrefix(buildPrefix(dest.getPath())) - .withDelimiter("/"); - return s3Client.listObjectsV2(listObjectsV2Request).getObjectSummaries().size() - + s3Client.listObjectsV2(listObjectsV2Request).getCommonPrefixes().size() - > 0; - } catch (AmazonS3Exception e) { - return false; - } - } - - public boolean existsFile(FsPath dest) { - try { - return s3Client.doesObjectExist(bucket, buildPrefix(dest.getPath(), false)); - } catch (AmazonS3Exception e) { - return false; - } - } - - @Override - public boolean delete(FsPath dest) throws IOException { - try { - ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); - listObjectsV2Request.withBucketName(bucket).withPrefix(buildPrefix(dest.getPath(), false)); - ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request); - String[] keyList = - result.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new); - DeleteObjectsRequest deleteObjectsRequest = - new DeleteObjectsRequest("test").withKeys(keyList); - s3Client.deleteObjects(deleteObjectsRequest); - return true; - } catch (AmazonS3Exception e) { - throw new IOException("You have not permission to access path " + dest.getPath()); - } - } - - @Override - public boolean renameTo(FsPath oldDest, FsPath newDest) throws IOException { - try { - String newOriginPath = buildPrefix(oldDest.getPath(), false); - String newDestPath = buildPrefix(newDest.getPath(), false); - ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); - listObjectsV2Request.withBucketName(bucket).withPrefix(newOriginPath); - ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request); - List keyList = - result.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey) - .collect(Collectors.toList()); - List newKeyList = - keyList.stream() - .map(key -> key.replaceFirst(newOriginPath, newDestPath)) - .collect(Collectors.toList()); - for (int i = 0; i < keyList.size(); i++) { - String key = keyList.get(i); - String newKey = newKeyList.get(i); - s3Client.copyObject(bucket, key, bucket, newKey); - s3Client.deleteObject(bucket, key); - } - return true; - } catch (AmazonS3Exception e) { - s3Client.deleteObject(bucket, newDest.getPath()); - throw new IOException( - "You have not permission to access path " - + oldDest.getPath() - + " or " - + newDest.getPath()); - } - } - - @Override - public boolean copy(String origin, String dest) throws IOException { - try { - String newOrigin = buildPrefix(origin, false); - String newDest = buildPrefix(dest, false); - ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); - listObjectsV2Request.withBucketName(bucket).withPrefix(newOrigin); - ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request); - List keyList = - result.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey) - .collect(Collectors.toList()); - List newKeyList = - keyList.stream() - .map(key -> key.replaceFirst(newOrigin, newDest)) - .collect(Collectors.toList()); - for (int i = 0; i < keyList.size(); i++) { - String key = keyList.get(i); - String newKey = newKeyList.get(i); - s3Client.copyObject(bucket, key, bucket, newKey); - } - return true; - } catch (AmazonS3Exception e) { - throw new IOException("You have not permission to access path " + origin + " or " + dest); - } - } - - private boolean isDir(S3ObjectSummary s3ObjectSummary, String prefix) { - return s3ObjectSummary.getKey().substring(prefix.length()).contains("/"); - } - - private boolean isInitFile(S3ObjectSummary s3ObjectSummary) { - return s3ObjectSummary.getKey().contains(INIT_FILE_NAME); - } - - @Override - public String listRoot() { - return "/"; - } - - @Override - public boolean mkdir(FsPath dest) throws IOException { - String path = new File(dest.getPath(), INIT_FILE_NAME).getPath(); - if (exists(new FsPath(path))) { - return false; - } - return create(path); - } - - @Override - public boolean mkdirs(FsPath dest) throws IOException { - return mkdir(dest); - } - - private FsPath fillStorageFile(FsPath fsPath, S3ObjectSummary s3ObjectSummary) { - fsPath.setModification_time(s3ObjectSummary.getLastModified().getTime()); - Owner owner = s3ObjectSummary.getOwner(); - if (owner != null) { - fsPath.setOwner(owner.getDisplayName()); - } - try { - fsPath.setIsdir(isDir(s3ObjectSummary, fsPath.getParent().getPath())); - } catch (Throwable e) { - logger.warn("Failed to fill storage file：" + fsPath.getPath(), e); - } - - if (fsPath.isdir()) { - fsPath.setLength(0); - } else { - fsPath.setLength(s3ObjectSummary.getSize()); - } - return fsPath; - } - - @Override - public boolean canRead(FsPath dest) { - return true; - } - - @Override - public boolean canRead(FsPath fsPath, String s) throws IOException { - return false; - } - - @Override - public boolean canWrite(FsPath dest) { - return true; - } - - @Override - public long getTotalSpace(FsPath dest) { - return 0; - } - - @Override - public long getFreeSpace(FsPath dest) { - return 0; - } - - @Override - public long getUsableSpace(FsPath dest) { - return 0; - } - - @Override - public boolean canExecute(FsPath dest) { - return true; - } - - @Override - public boolean setOwner(FsPath dest, String user, String group) { - return false; - } - - @Override - public boolean setOwner(FsPath dest, String user) { - return false; - } - - @Override - public boolean setGroup(FsPath dest, String group) { - return false; - } - - @Override - public boolean setPermission(FsPath dest, String permission) { - return false; - } - - @Override - public void close() throws IOException {} - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String buildPath(String path) { - if (path == null || "".equals(path)) return ""; - if (path.startsWith("/")) { - return StorageUtils.S3_SCHEMA + path; - } - return StorageUtils.S3_SCHEMA + "/" + path; - } - - public String buildPrefix(String path, boolean addTail) { - String res = path; - if (path == null || "".equals(path)) return ""; - if (path.startsWith("/")) { - res = path.replaceFirst("/", ""); - } - if (!path.endsWith("/") && addTail) { - res = res + "/"; - } - return res; - } - - public String buildPrefix(String path) { - return buildPrefix(path, true); - } -} - -class S3OutputStream extends ByteArrayOutputStream { - private AmazonS3 s3Client; - private String bucket; - private String path; - - public S3OutputStream(AmazonS3 s3Client, String bucket, String path) { - this.s3Client = s3Client; - this.bucket = bucket; - this.path = path; - } - - @Override - public void close() throws IOException { - byte[] buffer = this.toByteArray(); - try (InputStream in = new ByteArrayInputStream(buffer)) { - s3Client.putObject(bucket, path, in, new ObjectMetadata()); - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorFactory.java deleted file mode 100644 index 8f0c4016d1..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOMethodInterceptorFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.io; - -import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary; -import org.apache.linkis.storage.exception.StorageWarnException; - -import org.springframework.cglib.proxy.MethodInterceptor; - -public class IOMethodInterceptorFactory { - - private static IOMethodInterceptorCreator interceptorCreator = null; - - private IOMethodInterceptorFactory() {} - - /** - * This method is called when ioClient is initialized. ioClient初始化时会调用该方法 - * - * @param interceptorCreator - */ - public static void register(IOMethodInterceptorCreator interceptorCreator) { - IOMethodInterceptorFactory.interceptorCreator = interceptorCreator; - } - - public static MethodInterceptor getIOMethodInterceptor(String fsName) - throws StorageWarnException { - if (IOMethodInterceptorFactory.interceptorCreator == null) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.MUST_REGISTER_TOM.getErrorCode(), - LinkisStorageErrorCodeSummary.MUST_REGISTER_TOM.getErrorDesc()); - } - return IOMethodInterceptorFactory.interceptorCreator.createIOMethodInterceptor(fsName); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java deleted file mode 100644 index da51a2f13d..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.storage.FSFactory; -import org.apache.linkis.storage.conf.LinkisStorageConf; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.utils.StorageConfiguration; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.text.MessageFormat; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.THE_FILE_IS_EMPTY; - -public class DefaultResultSetFactory implements ResultSetFactory { - - private static final Logger logger = LoggerFactory.getLogger(DefaultResultSetFactory.class); - - private final Map>> resultClasses; - - private final String[] resultTypes; - - public DefaultResultSetFactory() { - resultClasses = - StorageUtils.loadClasses( - StorageConfiguration.STORAGE_RESULT_SET_CLASSES.getValue(), - StorageConfiguration.STORAGE_RESULT_SET_PACKAGE.getValue(), - t -> { - try { - return t.newInstance().resultSetType().toLowerCase(Locale.getDefault()); - } catch (InstantiationException e) { - logger.warn("DefaultResultSetFactory init failed", e); - } catch (IllegalAccessException e) { - logger.warn("DefaultResultSetFactory init failed", e); - } - return null; - }); - resultTypes = ResultSetFactory.resultSetType.keySet().toArray(new String[0]); - } - - @Override - public ResultSet getResultSetByType(String resultSetType) { - if (!resultClasses.containsKey(resultSetType)) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorCode(), - MessageFormat.format( - LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorDesc(), resultSetType)); - } - try { - return resultClasses.get(resultSetType).newInstance(); - } catch (InstantiationException | IllegalAccessException e) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorCode(), - MessageFormat.format( - LinkisStorageErrorCodeSummary.UNSUPPORTED_RESULT.getErrorDesc(), resultSetType), - e); - } - } - - @Override - public ResultSet getResultSetByPath(FsPath fsPath) - throws StorageWarnException { - return getResultSetByPath(fsPath, StorageUtils.getJvmUser()); - } - - @Override - public ResultSet getResultSetByContent(String content) { - return getResultSetByType(Dolphin.getType(content)); - } - - @Override - public boolean exists(String resultSetType) { - return resultClasses.containsKey(resultSetType); - } - - @Override - public boolean isResultSetPath(String path) { - return path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX) - || path.endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX) - || path.endsWith(LinkisStorageConf.ORC_FILE_SUFFIX); - } - - @Override - public boolean isResultSet(String content) { - try { - return resultClasses.containsKey(Dolphin.getType(content)); - } catch (Exception e) { - logger.info("Wrong result Set: " + e.getMessage()); - return false; - } - } - - @Override - public ResultSet getResultSet(String output) - throws StorageWarnException { - return getResultSet(output, StorageUtils.getJvmUser()); - } - - @Override - public String[] getResultSetType() { - return Arrays.copyOf(resultTypes, resultTypes.length); - } - - @Override - public ResultSet getResultSetByPath(FsPath fsPath, Fs fs) { - ResultSet resultSet = null; - try (InputStream inputStream = fs.read(fsPath)) { - if (fsPath.getPath().endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)) { - String resultSetType = Dolphin.getType(inputStream); - if (StringUtils.isEmpty(resultSetType)) { - throw new StorageWarnException( - THE_FILE_IS_EMPTY.getErrorCode(), - MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc(), fsPath.getPath())); - } - // Utils.tryQuietly(fs::close); - resultSet = getResultSetByType(resultSetType); - } else if (fsPath.getPath().endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX) - || fsPath.getPath().endsWith(LinkisStorageConf.ORC_FILE_SUFFIX)) { - resultSet = getResultSetByType(ResultSetFactory.TABLE_TYPE); - } - return resultSet; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public ResultSet getResultSetByPath( - FsPath fsPath, String proxyUser) { - if (fsPath == null) { - return null; - } - logger.info("Get Result Set By Path:" + fsPath.getPath()); - try (Fs fs = FSFactory.getFsByProxyUser(fsPath, proxyUser)) { - fs.init(new HashMap<>()); - try (InputStream inputStream = fs.read(fsPath)) { - String resultSetType = Dolphin.getType(inputStream); - if (StringUtils.isEmpty(resultSetType)) { - throw new StorageWarnException( - THE_FILE_IS_EMPTY.getErrorCode(), - MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc(), fsPath.getPath())); - } - IOUtils.closeQuietly(inputStream); - return getResultSetByType(resultSetType); - } catch (IOException e) { - throw new RuntimeException(e); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public ResultSet getResultSet( - String output, String proxyUser) { - if (isResultSetPath(output)) { - return getResultSetByPath(new FsPath(output), proxyUser); - } else if (isResultSet(output)) { - return getResultSetByContent(output); - } else { - return null; - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java deleted file mode 100644 index 249e326cde..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.domain.DataType; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.orc.OrcFile; -import org.apache.orc.Reader; -import org.apache.orc.RecordReader; -import org.apache.orc.TypeDescription; -import org.apache.orc.storage.ql.exec.vector.*; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class OrcResultSetReader - extends ResultSetReader { - - private static final Logger logger = LoggerFactory.getLogger(OrcResultSetReader.class); - - private FsPath fsPath; - - private final ResultSet resultSet; - - private final InputStream inputStream; - - private MetaData metaData; - - private Record row; - - private RecordReader rows; - - private Reader reader; - - public OrcResultSetReader(ResultSet resultSet, InputStream inputStream, FsPath fsPath) - throws IOException { - super(resultSet, inputStream); - this.resultSet = resultSet; - this.inputStream = inputStream; - this.fsPath = fsPath; - this.reader = - OrcFile.createReader( - new Path(fsPath.getPath()), OrcFile.readerOptions(new Configuration())); - this.rows = reader.rows(); - } - - @Override - public MetaData getMetaData() { - if (metaData == null) { - try { - List fieldNames = reader.getSchema().getFieldNames(); - List typeDescriptions = reader.getSchema().getChildren(); - List columnList = new ArrayList<>(); - for (int i = 0; i < fieldNames.size(); i++) { - Column column = - new Column( - fieldNames.get(i), - DataType.toDataType(typeDescriptions.get(i).getCategory().getName()), - ""); - columnList.add(column); - } - - metaData = new TableMetaData(columnList.toArray(new Column[0])); - } catch (Exception e) { - throw new RuntimeException("Failed to read parquet schema", e); - } - } - return metaData; - } - - @Override - public int skip(int recordNum) throws IOException { - if (recordNum < 0) return -1; - - for (int i = recordNum; i > 0; i--) { - try { - hasNext(); - } catch (Throwable t) { - return recordNum - i; - } - } - return recordNum; - } - - @Override - public long getPosition() throws IOException { - throw new UnsupportedOperationException("Storeage Unsupported type: getPosition"); - } - - @Override - public long available() throws IOException { - throw new UnsupportedOperationException("Storeage Unsupported type: available"); - } - - @Override - public boolean hasNext() throws IOException { - if (metaData == null) getMetaData(); - if (rows == null) return false; - VectorizedRowBatch batch = - reader.getSchema().createRowBatch(TypeDescription.RowBatchVersion.ORIGINAL, 1); - TableMetaData tableMetaData = (TableMetaData) metaData; - - if (rows.nextBatch(batch)) { - int rowNum = 0; - Object[] rowData = new Object[tableMetaData.getColumns().length]; - for (int i = 0; i < batch.numCols; i++) { - ColumnVector columnVector = batch.cols[i]; - if (columnVector instanceof BytesColumnVector) { - BytesColumnVector vector = (BytesColumnVector) columnVector; - rowData[i] = vector.toString(rowNum); - } else if (columnVector instanceof Decimal64ColumnVector) { - Decimal64ColumnVector vector = (Decimal64ColumnVector) columnVector; - rowData[i] = vector.vector[rowNum]; - } else if (columnVector instanceof DecimalColumnVector) { - DecimalColumnVector vector = (DecimalColumnVector) columnVector; - rowData[i] = vector.vector[rowNum]; - } else if (columnVector instanceof DoubleColumnVector) { - DoubleColumnVector vector = (DoubleColumnVector) columnVector; - rowData[i] = vector.vector[rowNum]; - } else if (columnVector instanceof ListColumnVector) { - ListColumnVector vector = (ListColumnVector) columnVector; - StringBuilder builder = new StringBuilder(); - vector.stringifyValue(builder, rowNum); - rowData[i] = builder.toString(); - } else if (columnVector instanceof IntervalDayTimeColumnVector) { - IntervalDayTimeColumnVector vector = (IntervalDayTimeColumnVector) columnVector; - StringBuilder builder = new StringBuilder(); - vector.stringifyValue(builder, rowNum); - rowData[i] = builder.toString(); - } else if (columnVector instanceof LongColumnVector) { - LongColumnVector vector = (LongColumnVector) columnVector; - rowData[i] = vector.vector[rowNum]; - } else if (columnVector instanceof MapColumnVector) { - MapColumnVector vector = (MapColumnVector) columnVector; - StringBuilder builder = new StringBuilder(); - vector.stringifyValue(builder, rowNum); - rowData[i] = builder.toString(); - } else if (columnVector instanceof MultiValuedColumnVector) { - MultiValuedColumnVector vector = (MultiValuedColumnVector) columnVector; - StringBuilder builder = new StringBuilder(); - vector.stringifyValue(builder, rowNum); - rowData[i] = builder.toString(); - } else if (columnVector instanceof StructColumnVector) { - StructColumnVector vector = (StructColumnVector) columnVector; - StringBuilder builder = new StringBuilder(); - vector.stringifyValue(builder, rowNum); - rowData[i] = builder.toString(); - } else if (columnVector instanceof TimestampColumnVector) { - TimestampColumnVector vector = (TimestampColumnVector) columnVector; - rowData[i] = vector.time[rowNum]; - } else if (columnVector instanceof UnionColumnVector) { - UnionColumnVector vector = (UnionColumnVector) columnVector; - StringBuilder builder = new StringBuilder(); - vector.stringifyValue(builder, rowNum); - rowData[i] = builder.toString(); - } - } - row = new TableRecord(rowData); - } else { - return false; - } - return row != null; - } - - @Override - public Record getRecord() { - if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)"); - if (row == null) { - throw new RuntimeException( - "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"); - } - return row; - } - - @Override - public void close() throws IOException { - IOUtils.closeQuietly(inputStream); - rows.close(); - reader.close(); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java deleted file mode 100644 index c4809f6499..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetWriter; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; -import org.apache.linkis.storage.utils.OrcUtils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.orc.CompressionKind; -import org.apache.orc.OrcFile; -import org.apache.orc.TypeDescription; -import org.apache.orc.Writer; -import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class OrcResultSetWriter - extends ResultSetWriter { - private static final Logger logger = LoggerFactory.getLogger(OrcResultSetWriter.class); - - private TypeDescription schema; - - private Writer writer; - - private boolean moveToWriteRow = false; - - private MetaData metaData = null; - - private final FsPath storePath; - - private final long maxCacheSize; - - private final ResultSet resultSet; - - public OrcResultSetWriter(ResultSet resultSet, long maxCacheSize, FsPath storePath) { - super(resultSet, maxCacheSize, storePath); - this.resultSet = resultSet; - this.maxCacheSize = maxCacheSize; - this.storePath = storePath; - } - - @Override - public void addMetaData(MetaData metaData) throws IOException { - if (!moveToWriteRow) { - this.metaData = metaData; - if (this.schema == null) { - this.schema = TypeDescription.createStruct(); - } - TableMetaData tableMetaData = (TableMetaData) this.metaData; - for (Column column : tableMetaData.columns) { - schema.addField(column.getColumnName(), OrcUtils.dataTypeToOrcType(column.getDataType())); - } - moveToWriteRow = true; - if (writer == null) { - writer = - OrcFile.createWriter( - new Path(storePath.getPath()), - OrcFile.writerOptions(new Configuration()) - .setSchema(schema) - .compress(CompressionKind.ZLIB) - .version(OrcFile.Version.V_0_12)); - } - } - } - - @Override - public void addRecord(Record record) { - if (moveToWriteRow) { - TableRecord tableRecord = (TableRecord) record; - TableMetaData tableMetaData = (TableMetaData) metaData; - try { - Object[] row = tableRecord.row; - VectorizedRowBatch batch = schema.createRowBatch(); - int rowCount = batch.size++; - - for (int i = 0; i < row.length; i++) { - OrcUtils.setColumn( - rowCount, batch.cols[i], tableMetaData.columns[i].getDataType(), row[i]); - if (batch.size == batch.getMaxSize()) { - writer.addRowBatch(batch); - batch.reset(); - } - } - writer.addRowBatch(batch); - - } catch (IOException e) { - logger.warn("addMetaDataAndRecordString failed", e); - } - } - } - - @Override - public FsPath toFSPath() { - return storePath; - } - - @Override - public String toString() { - return storePath.getSchemaPath(); - } - - @Override - public void addMetaDataAndRecordString(String content) {} - - @Override - public void addRecordString(String content) {} - - @Override - public void close() throws IOException { - writer.close(); - } - - @Override - public void flush() throws IOException {} -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java deleted file mode 100644 index c09804294d..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.domain.DataType; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; - -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.fs.Path; -import org.apache.parquet.avro.AvroParquetReader; -import org.apache.parquet.hadoop.ParquetReader; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ParquetResultSetReader - extends ResultSetReader { - - private static final Logger logger = LoggerFactory.getLogger(ParquetResultSetReader.class); - - private FsPath fsPath; - - private final ResultSet resultSet; - - private final InputStream inputStream; - - private MetaData metaData; - - private Record row; - - private ParquetReader parquetReader; - - private GenericRecord record; - - public ParquetResultSetReader(ResultSet resultSet, InputStream inputStream, FsPath fsPath) - throws IOException { - super(resultSet, inputStream); - this.resultSet = resultSet; - this.inputStream = inputStream; - this.fsPath = fsPath; - this.parquetReader = - AvroParquetReader.builder(new Path(fsPath.getPath())).build(); - this.record = parquetReader.read(); - } - - @Override - public MetaData getMetaData() { - if (metaData == null) { - try { - List fields = record.getSchema().getFields(); - List columnList = - fields.stream() - .map( - field -> - new Column( - field.name(), - DataType.toDataType(field.schema().getType().getName()), - "")) - .collect(Collectors.toList()); - - metaData = new TableMetaData(columnList.toArray(new Column[0])); - } catch (Exception e) { - throw new RuntimeException("Failed to read parquet schema", e); - } - } - return metaData; - } - - @Override - public int skip(int recordNum) throws IOException { - if (recordNum < 0) return -1; - - for (int i = recordNum; i > 0; i--) { - try { - this.record = parquetReader.read(); - } catch (Throwable t) { - return recordNum - i; - } - } - return recordNum; - } - - @Override - public long getPosition() throws IOException { - throw new UnsupportedOperationException("Storeage Unsupported type: getPosition"); - } - - @Override - public long available() throws IOException { - throw new UnsupportedOperationException("Storeage Unsupported type: available"); - } - - @Override - public boolean hasNext() throws IOException { - if (metaData == null) getMetaData(); - if (record == null) return false; - ArrayList resultList = new ArrayList<>(); - TableMetaData tableMetaData = (TableMetaData) metaData; - int length = tableMetaData.getColumns().length; - for (int i = 0; i < length; i++) { - resultList.add(record.get(i)); - } - row = new TableRecord(resultList.toArray(new Object[0])); - if (row == null) return false; - return record != null; - } - - @Override - public Record getRecord() { - if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)"); - if (row == null) { - throw new RuntimeException( - "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"); - } - try { - this.record = parquetReader.read(); - } catch (IOException e) { - throw new RuntimeException("Failed to read parquet record", e); - } - return row; - } - - @Override - public void close() throws IOException { - IOUtils.closeQuietly(inputStream); - parquetReader.close(); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java deleted file mode 100644 index 6fbac3c8cb..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetWriter; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; - -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.Path; -import org.apache.parquet.avro.AvroParquetWriter; -import org.apache.parquet.hadoop.ParquetFileWriter; -import org.apache.parquet.hadoop.ParquetWriter; -import org.apache.parquet.hadoop.metadata.CompressionCodecName; - -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ParquetResultSetWriter - extends ResultSetWriter { - private static final Logger logger = LoggerFactory.getLogger(ParquetResultSetWriter.class); - - private Schema schema; - - private ParquetWriter parquetWriter; - - private boolean moveToWriteRow = false; - - private MetaData metaData = null; - - private final FsPath storePath; - - private final long maxCacheSize; - - private final ResultSet resultSet; - - public ParquetResultSetWriter(ResultSet resultSet, long maxCacheSize, FsPath storePath) { - super(resultSet, maxCacheSize, storePath); - this.resultSet = resultSet; - this.maxCacheSize = maxCacheSize; - this.storePath = storePath; - } - - @Override - public void addMetaData(MetaData metaData) throws IOException { - if (!moveToWriteRow) { - this.metaData = metaData; - SchemaBuilder.FieldAssembler fieldAssembler = SchemaBuilder.record("linkis").fields(); - TableMetaData tableMetaData = (TableMetaData) this.metaData; - for (Column column : tableMetaData.columns) { - fieldAssembler - .name(column.getColumnName().replaceAll("\\.", "_").replaceAll("[^a-zA-Z0-9_]", "")) - .doc(column.getComment()) - .type(column.getDataType().getTypeName().toLowerCase()) - .noDefault(); - } - schema = fieldAssembler.endRecord(); - moveToWriteRow = true; - if (parquetWriter == null) { - parquetWriter = - AvroParquetWriter.builder(new Path(storePath.getPath())) - .withSchema(schema) - .withCompressionCodec(CompressionCodecName.SNAPPY) - .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) - .build(); - } - } - } - - @Override - public void addRecord(Record record) { - if (moveToWriteRow) { - TableRecord tableRecord = (TableRecord) record; - try { - Object[] row = tableRecord.row; - GenericRecord genericRecord = new GenericData.Record(schema); - for (int i = 0; i < row.length; i++) { - genericRecord.put(schema.getFields().get(i).name(), row[i]); - } - parquetWriter.write(genericRecord); - } catch (IOException e) { - logger.warn("addMetaDataAndRecordString failed", e); - } - } - } - - @Override - public FsPath toFSPath() { - return storePath; - } - - @Override - public String toString() { - return storePath.getSchemaPath(); - } - - @Override - public void addMetaDataAndRecordString(String content) {} - - @Override - public void addRecordString(String content) {} - - @Override - public void close() throws IOException { - parquetWriter.close(); - } - - @Override - public void flush() throws IOException {} -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetFactory.java deleted file mode 100644 index ed65cea16c..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetFactory.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.*; -import org.apache.linkis.common.io.resultset.ResultSet; - -import java.util.LinkedHashMap; -import java.util.Map; - -public interface ResultSetFactory { - - String TEXT_TYPE = "1"; - String TABLE_TYPE = "2"; - String IO_TYPE = "3"; - String PICTURE_TYPE = "4"; - String HTML_TYPE = "5"; - - /** TODO 修改为注册形式，并修改ResultSet的getResultType逻辑 Result set corresponding type record(结果集对应类型记录) */ - Map resultSetType = - new LinkedHashMap() { - { - put(TEXT_TYPE, "TEXT"); - put(TABLE_TYPE, "TABLE"); - put(IO_TYPE, "IO"); - put(PICTURE_TYPE, "PICTURE"); - put(HTML_TYPE, "HTML"); - } - }; - - DefaultResultSetFactory factory = new DefaultResultSetFactory(); - - static ResultSetFactory getInstance() { - return factory; - } - - ResultSet getResultSetByType(String resultSetType); - - ResultSet getResultSetByPath(FsPath fsPath); - - ResultSet getResultSetByPath(FsPath fsPath, Fs fs); - - ResultSet getResultSetByContent(String content); - - boolean exists(String resultSetType); - - boolean isResultSetPath(String path); - - boolean isResultSet(String content); - - ResultSet getResultSet(String output); - - ResultSet getResultSetByPath( - FsPath fsPath, String proxyUser); - - ResultSet getResultSet(String output, String proxyUser); - - String[] getResultSetType(); -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java deleted file mode 100644 index bf46f49f9b..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.storage.FSFactory; -import org.apache.linkis.storage.conf.LinkisStorageConf; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary; -import org.apache.linkis.storage.exception.StorageErrorCode; -import org.apache.linkis.storage.exception.StorageReadException; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; -import org.apache.linkis.storage.resultset.table.TableResultSet; - -import java.io.IOException; -import java.io.InputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ResultSetReaderFactory { - private static final Logger logger = LoggerFactory.getLogger(ResultSetReaderFactory.class); - - public static ResultSetReader getResultSetReader( - ResultSet resultSet, InputStream inputStream, FsPath fsPath) { - ResultSetReader resultSetReader = null; - if (fsPath.getPath().endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)) { - resultSetReader = new StorageResultSetReader<>(resultSet, inputStream); - } else if (fsPath.getPath().endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX)) { - try { - resultSetReader = new ParquetResultSetReader<>(resultSet, inputStream, fsPath); - } catch (IOException e) { - throw new StorageReadException( - StorageErrorCode.READ_PARQUET_FAILED.getCode(), - StorageErrorCode.READ_PARQUET_FAILED.getMessage(), - e); - } - } else if (fsPath.getPath().endsWith(LinkisStorageConf.ORC_FILE_SUFFIX)) { - try { - resultSetReader = new OrcResultSetReader<>(resultSet, inputStream, fsPath); - } catch (IOException e) { - throw new StorageReadException( - StorageErrorCode.READ_ORC_FAILED.getCode(), - StorageErrorCode.READ_ORC_FAILED.getMessage(), - e); - } - } - return resultSetReader; - } - - public static ResultSetReader getResultSetReader( - ResultSet resultSet, String value) { - return new StorageResultSetReader<>(resultSet, value); - } - - public static ResultSetReader getResultSetReader(String res) throws IOException { - ResultSetFactory rsFactory = ResultSetFactory.getInstance(); - if (rsFactory.isResultSet(res)) { - ResultSet resultSet = rsFactory.getResultSet(res); - return ResultSetReaderFactory.getResultSetReader(resultSet, res); - } else { - FsPath resPath = new FsPath(res); - ResultSet resultSet = - rsFactory.getResultSetByPath(resPath); - Fs fs = FSFactory.getFs(resPath); - fs.init(null); - ResultSetReader reader = - ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath), resPath); - if (reader instanceof StorageResultSetReader) { - ((StorageResultSetReader) reader).setFs(fs); - } - return (StorageResultSetReader) reader; - } - } - - public static ResultSetReader getTableResultReader(String res) { - ResultSetFactory rsFactory = ResultSetFactory.getInstance(); - if (rsFactory.isResultSet(res)) { - ResultSet resultSet = rsFactory.getResultSet(res); - if (!ResultSetFactory.TABLE_TYPE.equals(resultSet.resultSetType())) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorCode(), - LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorDesc()); - } - return ResultSetReaderFactory.getResultSetReader( - (TableResultSet) resultSet, res); - } else { - FsPath resPath = new FsPath(res); - ResultSet resultSet = rsFactory.getResultSetByPath(resPath); - if (!ResultSetFactory.TABLE_TYPE.equals(resultSet.resultSetType())) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorCode(), - LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorDesc()); - } - - Fs fs = FSFactory.getFs(resPath); - logger.info("Try to init Fs with path:{}", resPath.getPath()); - try { - fs.init(null); - InputStream read = fs.read(resPath); - - return ResultSetReaderFactory.getResultSetReader( - (TableResultSet) resultSet, read, resPath); - } catch (IOException e) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorCode(), - LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorDesc()); - } - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java deleted file mode 100644 index 9cb1999367..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.common.io.resultset.ResultSetWriter; -import org.apache.linkis.storage.conf.LinkisStorageConf; -import org.apache.linkis.storage.resultset.table.TableResultSet; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ResultSetWriterFactory { - private static final Logger logger = LoggerFactory.getLogger(ResultSetWriterFactory.class); - - public static ResultSetWriter getResultSetWriter( - ResultSet resultSet, long maxCacheSize, FsPath storePath) { - String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE; - ResultSetWriter writer = null; - if (engineResultType.equals(LinkisStorageConf.PARQUET) && resultSet instanceof TableResultSet) { - writer = new ParquetResultSetWriter<>(resultSet, maxCacheSize, storePath); - } else if (engineResultType.equals(LinkisStorageConf.ORC) - && resultSet instanceof TableResultSet) { - writer = new OrcResultSetWriter<>(resultSet, maxCacheSize, storePath); - } else { - writer = new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath); - } - return writer; - } - - public static ResultSetWriter getResultSetWriter( - ResultSet resultSet, long maxCacheSize, FsPath storePath, String proxyUser) { - String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE; - ResultSetWriter writer = null; - if (engineResultType.equals(LinkisStorageConf.PARQUET) && resultSet instanceof TableResultSet) { - writer = new ParquetResultSetWriter<>(resultSet, maxCacheSize, storePath); - } else if (engineResultType.equals(LinkisStorageConf.ORC) - && resultSet instanceof TableResultSet) { - writer = new OrcResultSetWriter<>(resultSet, maxCacheSize, storePath); - } else { - writer = new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath); - StorageResultSetWriter storageResultSetWriter = (StorageResultSetWriter) writer; - storageResultSetWriter.setProxyUser(proxyUser); - } - return writer; - } - - public static Record[] getRecordByWriter( - ResultSetWriter writer, long limit) throws IOException { - String res = writer.toString(); - return getRecordByRes(res, limit); - } - - public static Record[] getRecordByRes(String res, long limit) throws IOException { - ResultSetReader reader = ResultSetReaderFactory.getResultSetReader(res); - int count = 0; - List records = new ArrayList<>(); - reader.getMetaData(); - while (reader.hasNext() && count < limit) { - records.add(reader.getRecord()); - count++; - } - return records.toArray(new Record[0]); - } - - public static Record getLastRecordByRes(String res) throws IOException { - ResultSetReader reader = ResultSetReaderFactory.getResultSetReader(res); - Record record = null; - try { - reader.getMetaData(); - while (reader.hasNext()) { - record = reader.getRecord(); - } - } catch (IOException e) { - logger.warn("ResultSetWriter getLastRecordByRes failed", e); - } - return record; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java deleted file mode 100644 index c708f5faf6..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.storage.conf.LinkisStorageConf; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.resultset.table.TableResultSet; -import org.apache.linkis.storage.utils.StorageConfiguration; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class StorageResultSet - implements ResultSet { - - private static final Logger logger = LoggerFactory.getLogger(StorageResultSet.class); - - private byte[] resultHeaderBytes = null; - - { - byte[] arr2 = Dolphin.getIntBytes(Integer.parseInt(resultSetType())); - byte[] mergedArray = new byte[Dolphin.MAGIC_BYTES.length + arr2.length]; - System.arraycopy(Dolphin.MAGIC_BYTES, 0, mergedArray, 0, Dolphin.MAGIC_BYTES.length); - System.arraycopy(arr2, 0, mergedArray, Dolphin.MAGIC_BYTES.length, arr2.length); - resultHeaderBytes = mergedArray; - } - - @Override - public String charset() { - return StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue(); - } - - @Override - public FsPath getResultSetPath(FsPath parentDir, String fileName) { - String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE; - String fileSuffix = Dolphin.DOLPHIN_FILE_SUFFIX; - if (engineResultType.equals(LinkisStorageConf.PARQUET) && this instanceof TableResultSet) { - fileSuffix = LinkisStorageConf.PARQUET_FILE_SUFFIX; - } else if (engineResultType.equals(LinkisStorageConf.ORC) && this instanceof TableResultSet) { - fileSuffix = LinkisStorageConf.ORC_FILE_SUFFIX; - } - final String path = - parentDir.getPath().endsWith("/") - ? parentDir.getUriString() + fileName + fileSuffix - : parentDir.getUriString() + "/" + fileName + fileSuffix; - logger.info("Get result set path: {}", path); - return new FsPath(path); - } - - @Override - public byte[] getResultSetHeader() { - return resultHeaderBytes; - } - - @Override - public boolean belongToPath(String path) { - return path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX); - } - - @Override - public boolean belongToResultSet(String content) { - try { - return Dolphin.getType(content).equals(resultSetType()); - } catch (Exception e) { - logger.info("Wrong result set: ", e); - return false; - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java deleted file mode 100644 index c0222cc848..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetReader.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class StorageResultSetReader - extends ResultSetReader { - - private static final Logger logger = LoggerFactory.getLogger(StorageResultSetReader.class); - - private final ResultSet resultSet; - private final InputStream inputStream; - private final ResultDeserializer deserializer; - private K metaData; - private Record row; - private int colCount = 0; - private int rowCount = 0; - private Fs fs; - - private final int READ_CACHE = 1024; - - public StorageResultSetReader(ResultSet resultSet, InputStream inputStream) { - super(resultSet, inputStream); - this.resultSet = resultSet; - this.inputStream = inputStream; - this.deserializer = resultSet.createResultSetDeserializer(); - } - - public StorageResultSetReader(ResultSet resultSet, String value) { - this(resultSet, new ByteArrayInputStream(value.getBytes(Dolphin.CHAR_SET))); - } - - public void init() throws IOException { - String resType = Dolphin.getType(inputStream); - if (!StringUtils.equals(resultSet.resultSetType(), resType)) { - throw new RuntimeException( - "File type does not match(文件类型不匹配): " - + ResultSetFactory.resultSetType.getOrDefault(resType, "TABLE")); - } - } - - public byte[] readLine() { - int rowLen = 0; - try { - rowLen = Dolphin.readInt(inputStream); - } catch (StorageWarnException | IOException e) { - logger.info("Read finished(读取完毕)"); - return null; - } - - int len = 0; - byte[] rowBuffer = null; - try { - rowBuffer = new byte[rowLen]; - len = StorageUtils.readBytes(inputStream, rowBuffer, rowLen); - } catch (OutOfMemoryError error) { - logger.error("Result set read oom, read size {} Byte", rowLen); - throw new RuntimeException(error); - } - if (len != rowLen) { - throw new RuntimeException( - "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"); - } - rowCount++; - return rowBuffer; - } - - @Override - public Record getRecord() { - if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)"); - if (row == null) { - throw new RuntimeException( - "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"); - } - return row; - } - - public void setFs(Fs fs) { - this.fs = fs; - } - - public Fs getFs() { - return fs; - } - - @Override - public MetaData getMetaData() { - if (metaData == null) { - try { - init(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - metaData = deserializer.createMetaData(readLine()); - return metaData; - } - - @Override - public int skip(int recordNum) throws IOException { - if (recordNum < 0) return -1; - - if (metaData == null) getMetaData(); - for (int i = recordNum; i > 0; i--) { - try { - inputStream.skip(Dolphin.readInt(inputStream)); - } catch (Throwable t) { - return recordNum - i; - } - } - return recordNum; - } - - @Override - public long getPosition() throws IOException { - return rowCount; - } - - @Override - public boolean hasNext() throws IOException { - if (metaData == null) getMetaData(); - byte[] line = readLine(); - if (line == null) return false; - row = deserializer.createRecord(line); - if (row == null) return false; - return true; - } - - @Override - public long available() throws IOException { - return inputStream.available(); - } - - @Override - public void close() throws IOException { - IOUtils.closeQuietly(inputStream); - if (this.fs != null) { - this.fs.close(); - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java deleted file mode 100644 index 230c68301c..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetWriter; -import org.apache.linkis.storage.FSFactory; -import org.apache.linkis.storage.conf.LinkisStorageConf; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.exception.StorageErrorException; -import org.apache.linkis.storage.utils.FileSystemUtils; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.exception.StorageErrorCode.FS_ERROR; - -public class StorageResultSetWriter - extends ResultSetWriter { - private static final Logger logger = LoggerFactory.getLogger(StorageResultSetWriter.class); - - private final ResultSet resultSet; - private final long maxCacheSize; - private final FsPath storePath; - - private final ResultSerializer serializer; - private boolean moveToWriteRow = false; - private OutputStream outputStream = null; - private int rowCount = 0; - private final List buffer = new ArrayList(); - private Fs fs = null; - private MetaData rMetaData = null; - private String proxyUser = StorageUtils.getJvmUser(); - private boolean fileCreated = false; - private boolean closed = false; - private final Object WRITER_LOCK_CREATE = new Object(); - private final Object WRITER_LOCK_CLOSE = new Object(); - - public StorageResultSetWriter(ResultSet resultSet, long maxCacheSize, FsPath storePath) { - super(resultSet, maxCacheSize, storePath); - this.resultSet = resultSet; - this.maxCacheSize = maxCacheSize; - this.storePath = storePath; - - this.serializer = resultSet.createResultSetSerializer(); - } - - public MetaData getMetaData() { - return rMetaData; - } - - public void setProxyUser(String proxyUser) { - this.proxyUser = proxyUser; - } - - public boolean isEmpty() { - return rMetaData == null && buffer.size() <= Dolphin.FILE_EMPTY; - } - - public void init() { - try { - writeLine(resultSet.getResultSetHeader(), true); - } catch (IOException e) { - logger.warn("StorageResultSetWriter init failed", e); - } - } - - public void createNewFile() { - if (!fileCreated) { - synchronized (WRITER_LOCK_CREATE) { - if (!fileCreated) { - if (storePath != null && outputStream == null) { - logger.info("Try to create a new file:{}, with proxy user:{}", storePath, proxyUser); - fs = FSFactory.getFsByProxyUser(storePath, proxyUser); - try { - fs.init(null); - FileSystemUtils.createNewFile(storePath, proxyUser, true); - outputStream = fs.write(storePath, true); - } catch (Exception e) { - throw new StorageErrorException( - FS_ERROR.getCode(), "StorageResultSetWriter createNewFile failed", e); - } - logger.info("Succeed to create a new file:{}", storePath); - fileCreated = true; - } - } - } - } else if (storePath != null && outputStream == null) { - logger.warn("outputStream had been set null, but createNewFile() was called again."); - } - } - - public void writeLine(byte[] bytes, boolean cache) throws IOException { - if (closed) { - logger.warn("the writer had been closed, but writeLine() was still called."); - return; - } - if (bytes.length > LinkisStorageConf.ROW_BYTE_MAX_LEN) { - throw new IOException( - String.format( - "A single row of data cannot exceed %s", LinkisStorageConf.ROW_BYTE_MAX_LEN_STR)); - } - if (buffer.size() > maxCacheSize && !cache) { - if (outputStream == null) { - createNewFile(); - } - flush(); - outputStream.write(bytes); - } else { - for (byte b : bytes) { - buffer.add(b); - } - } - } - - @Override - public String toString() { - if (outputStream == null) { - if (isEmpty()) { - return ""; - } - - byte[] byteArray = getBytes(); - return new String(byteArray, Dolphin.CHAR_SET); - } - return storePath.getSchemaPath(); - } - - private byte[] getBytes() { - byte[] byteArray = new byte[buffer.size()]; - for (int i = 0; i < buffer.size(); i++) { - byteArray[i] = buffer.get(i); - } - return byteArray; - } - - @Override - public FsPath toFSPath() { - return storePath; - } - - @Override - public void addMetaDataAndRecordString(String content) { - if (!moveToWriteRow) { - byte[] bytes = content.getBytes(Dolphin.CHAR_SET); - try { - writeLine(bytes, false); - } catch (IOException e) { - logger.warn("addMetaDataAndRecordString failed", e); - } - } - moveToWriteRow = true; - } - - @Override - public void addRecordString(String content) {} - - @Override - public void addMetaData(MetaData metaData) throws IOException { - if (!moveToWriteRow) { - rMetaData = metaData; - init(); - if (metaData == null) { - writeLine(serializer.metaDataToBytes(metaData), true); - } else { - writeLine(serializer.metaDataToBytes(metaData), false); - } - moveToWriteRow = true; - } - } - - @Override - public void addRecord(Record record) { - if (moveToWriteRow) { - rowCount++; - try { - writeLine(serializer.recordToBytes(record), false); - } catch (IOException e) { - logger.warn("addMetaDataAndRecordString failed", e); - } - } - } - - public void closeFs() { - if (fs != null) { - IOUtils.closeQuietly(fs); - fs = null; - } - } - - @Override - public void close() { - if (closed) { - logger.warn("the writer had been closed, but close() was still called."); - return; - } - synchronized (WRITER_LOCK_CLOSE) { - if (!closed) { - closed = true; - } else { - return; - } - } - try { - if (outputStream != null) { - flush(); - } - } finally { - if (outputStream != null) { - IOUtils.closeQuietly(outputStream); - outputStream = null; - } - closeFs(); - } - } - - @Override - public void flush() { - createNewFile(); - if (outputStream != null) { - try { - if (!buffer.isEmpty()) { - outputStream.write(getBytes()); - buffer.clear(); - } - if (outputStream instanceof HdfsDataOutputStream) { - ((HdfsDataOutputStream) outputStream).hflush(); - } else { - outputStream.flush(); - } - } catch (IOException e) { - logger.warn("Error encountered when flush result set", e); - } - } - if (closed && logger.isDebugEnabled()) { - logger.debug("the writer had been closed, but flush() was still called."); - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/html/HtmlResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/html/HtmlResultSet.java deleted file mode 100644 index 00c0e7b2a7..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/html/HtmlResultSet.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.html; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.StorageResultSet; -import org.apache.linkis.storage.resultset.txt.TextResultDeserializer; -import org.apache.linkis.storage.resultset.txt.TextResultSerializer; - -import java.io.Serializable; - -public class HtmlResultSet extends StorageResultSet - implements Serializable { - - @Override - public String resultSetType() { - return ResultSetFactory.HTML_TYPE; - } - - @Override - public ResultSerializer createResultSetSerializer() { - return new TextResultSerializer(); - } - - @Override - public ResultDeserializer createResultSetDeserializer() { - return new TextResultDeserializer(); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultDeserializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultDeserializer.java deleted file mode 100644 index 02d83aa7bb..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultDeserializer.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.io; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.storage.domain.Dolphin; - -import org.apache.commons.codec.binary.Base64; - -public class IOResultDeserializer extends ResultDeserializer { - - @Override - public IOMetaData createMetaData(byte[] bytes) { - String[] values = Dolphin.getString(bytes, 0, bytes.length).split(Dolphin.COL_SPLIT); - return new IOMetaData(Integer.parseInt(values[0]), Integer.parseInt(values[1])); - } - - @Override - public IORecord createRecord(byte[] bytes) { - return new IORecord(Base64.decodeBase64(Dolphin.getString(bytes, 0, bytes.length))); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSerializer.java deleted file mode 100644 index 2401e361a5..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSerializer.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.io; - -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.codec.binary.Base64; - -public class IOResultSerializer extends ResultSerializer { - - @Override - public byte[] metaDataToBytes(MetaData metaData) { - IOMetaData ioMetaData = (IOMetaData) metaData; - return lineToBytes(ioMetaData.off + Dolphin.COL_SPLIT + ioMetaData.len); - } - - @Override - public byte[] recordToBytes(Record record) { - IORecord ioRecord = (IORecord) record; - return lineToBytes(Base64.encodeBase64String(ioRecord.value)); - } - - private byte[] lineToBytes(String value) { - byte[] bytes = value == null ? Dolphin.NULL_BYTES : Dolphin.getBytes(value); - byte[] intBytes = Dolphin.getIntBytes(bytes.length); - return StorageUtils.mergeByteArrays(intBytes, bytes); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSet.java deleted file mode 100644 index 67f8e76904..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/io/IOResultSet.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.io; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.StorageResultSet; - -import java.io.Serializable; - -public class IOResultSet extends StorageResultSet implements Serializable { - - @Override - public String resultSetType() { - return ResultSetFactory.IO_TYPE; - } - - @Override - public ResultSerializer createResultSetSerializer() { - return new IOResultSerializer(); - } - - @Override - public ResultDeserializer createResultSetDeserializer() { - return new IOResultDeserializer(); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/picture/PictureResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/picture/PictureResultSet.java deleted file mode 100644 index 5e73592a7b..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/picture/PictureResultSet.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.picture; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.StorageResultSet; -import org.apache.linkis.storage.resultset.txt.TextResultDeserializer; -import org.apache.linkis.storage.resultset.txt.TextResultSerializer; - -import java.io.Serializable; - -public class PictureResultSet extends StorageResultSet - implements Serializable { - - @Override - public String resultSetType() { - return ResultSetFactory.PICTURE_TYPE; - } - - @Override - public ResultSerializer createResultSetSerializer() { - return new TextResultSerializer(); - } - - @Override - public ResultDeserializer createResultSetDeserializer() { - return new TextResultDeserializer(); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java deleted file mode 100644 index 7e1d6c35fe..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultDeserializer.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.table; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.domain.DataType; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.exception.StorageWarnException; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.PARSING_METADATA_FAILED; - -public class TableResultDeserializer extends ResultDeserializer { - private static final Logger logger = LoggerFactory.getLogger(TableResultDeserializer.class); - - private TableMetaData metaData; - - @Override - public TableMetaData createMetaData(byte[] bytes) { - int colByteLen = Integer.parseInt(Dolphin.getString(bytes, 0, Dolphin.INT_LEN)); - String colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen); - String[] colArray = - colString.endsWith(Dolphin.COL_SPLIT) - ? colString.substring(0, colString.length() - 1).split(Dolphin.COL_SPLIT) - : colString.split(Dolphin.COL_SPLIT); - int index = Dolphin.INT_LEN + colByteLen; - if (colArray.length % 3 != 0) { - throw new StorageWarnException( - PARSING_METADATA_FAILED.getErrorCode(), PARSING_METADATA_FAILED.getErrorDesc()); - } - List columns = new ArrayList<>(); - for (int i = 0; i < colArray.length; i += 3) { - int len = Integer.parseInt(colArray[i]); - String colName = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)); - index += len; - len = Integer.parseInt(colArray[i + 1]); - String colType = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)); - index += len; - len = Integer.parseInt(colArray[i + 2]); - String colComment = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)); - index += len; - columns.add(new Column(colName, DataType.toDataType(colType), colComment)); - } - metaData = new TableMetaData(columns.toArray(new Column[0])); - return metaData; - } - - /** - * colByteLen:All column fields are long(所有列字段长 记录的长度) colString：Obtain column - * length(获得列长)：10，20，21 colArray：Column length array(列长数组) Get data by column length(通过列长获得数据) - * - * @param bytes - * @return - */ - @Override - public TableRecord createRecord(byte[] bytes) { - int colByteLen = Integer.parseInt(Dolphin.getString(bytes, 0, Dolphin.INT_LEN)); - String colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen); - String[] colArray; - if (colString.endsWith(Dolphin.COL_SPLIT)) { - colArray = colString.substring(0, colString.length() - 1).split(Dolphin.COL_SPLIT); - } else { - colArray = colString.split(Dolphin.COL_SPLIT); - } - int index = Dolphin.INT_LEN + colByteLen; - Object[] data = new Object[colArray.length]; - for (int i = 0; i < colArray.length; i++) { - int len = Integer.parseInt(colArray[i]); - String res = Dolphin.getString(bytes, index, len); - index += len; - if (i >= metaData.columns.length) { - data[i] = res; - } else { - data[i] = DataType.toValue(metaData.columns[i].getDataType(), res); - } - } - return new TableRecord(data); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java deleted file mode 100644 index 5f40aa33f3..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSerializer.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.table; - -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.domain.Dolphin; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class TableResultSerializer extends ResultSerializer { - - @Override - public byte[] metaDataToBytes(MetaData metaData) { - TableMetaData tableMetaData = (TableMetaData) metaData; - Object[] objects = - Arrays.stream(tableMetaData.columns).map(Column::toArray).flatMap(Arrays::stream).toArray(); - return lineToBytes(objects); - } - - @Override - public byte[] recordToBytes(Record record) { - TableRecord tableRecord = (TableRecord) record; - return lineToBytes(tableRecord.row); - } - - /** - * Convert a row of data to an array of Bytes Convert the data to byte and get the corresponding - * total byte length to write to the file Data write format: line length (fixed length) column - * length (fixed length) field index comma segmentation real data For example: - * 000000004900000000116,10,3,4,5,peace1johnnwang1101true11.51 The length of the line does not - * include its own length 将一行数据转换为Bytes的数组 对数据转换为byte，并获取相应的总byte长度写入文件 数据写入格式：行长(固定长度) 列长(固定长度) - * 字段索引逗号分割 真实数据 如：000000004900000000116,10,3,4,5,peace1johnnwang1101true11.51 其中行长不包括自身长度 - * - * @param line - */ - private byte[] lineToBytes(Object[] line) { - // Data cache(数据缓存) - List dataBytes = new ArrayList<>(); - // Column cache(列缓存) - List colIndex = new ArrayList<>(); - int colByteLen = 0; - int length = 0; - for (Object data : line) { - byte[] bytes = data == null ? Dolphin.LINKIS_NULL_BYTES : Dolphin.getBytes(data); - dataBytes.add(bytes); - byte[] colBytes = Dolphin.getBytes(bytes.length); - colIndex.add(colBytes); - colIndex.add(Dolphin.COL_SPLIT_BYTES); - colByteLen += colBytes.length + Dolphin.COL_SPLIT_LEN; - length += bytes.length; - } - length += colByteLen + Dolphin.INT_LEN; - return toByteArray(length, colByteLen, colIndex, dataBytes); - } - - /** - * Splice a row of data into a byte array(将一行的数据拼接成byte数组) - * - * @param length The total length of the line data byte, excluding its own - * length(行数据byte总长度，不包括自身的长度) - * @param colByteLen Record field index byte column length(记录字段索引byte的列长) - * @param colIndex Field index, including separator comma(字段索引，包括分割符逗号) - * @param dataBytes Byte of real data(真实数据的byte) - * @return - */ - public static byte[] toByteArray( - int length, int colByteLen, List colIndex, List dataBytes) { - List row = new ArrayList<>(); - colIndex.addAll(dataBytes); - - for (byte intByte : Dolphin.getIntBytes(length)) { - row.add(intByte); - } - - for (byte colByte : Dolphin.getIntBytes(colByteLen)) { - row.add(colByte); - } - - colIndex.forEach( - bytes -> { - for (byte b : bytes) { - row.add(b); - } - }); - byte[] result = new byte[row.size()]; - for (int i = 0; i < result.length; i++) { - result[i] = row.get(i); - } - return result; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSet.java deleted file mode 100644 index 5f356af0ca..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableResultSet.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.table; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.StorageResultSet; - -import java.io.Serializable; - -public class TableResultSet extends StorageResultSet - implements Serializable { - - @Override - public String resultSetType() { - return ResultSetFactory.TABLE_TYPE; - } - - @Override - public ResultSerializer createResultSetSerializer() { - return new TableResultSerializer(); - } - - @Override - public ResultDeserializer createResultSetDeserializer() { - return new TableResultDeserializer(); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.java deleted file mode 100644 index 3165e5af86..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.txt; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.domain.Dolphin; - -public class TextResultDeserializer extends ResultDeserializer { - - @Override - public LineMetaData createMetaData(byte[] bytes) { - return new LineMetaData(Dolphin.getString(bytes, 0, bytes.length)); - } - - @Override - public LineRecord createRecord(byte[] bytes) { - return new LineRecord(Dolphin.getString(bytes, 0, bytes.length)); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSerializer.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSerializer.java deleted file mode 100644 index 5555ad9eba..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSerializer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.txt; - -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.utils.StorageUtils; - -public class TextResultSerializer extends ResultSerializer { - - @Override - public byte[] metaDataToBytes(MetaData metaData) { - if (metaData == null) { - return lineToBytes(null); - } else { - LineMetaData textMetaData = (LineMetaData) metaData; - return lineToBytes(textMetaData.getMetaData()); - } - } - - @Override - public byte[] recordToBytes(Record record) { - LineRecord textRecord = (LineRecord) record; - return lineToBytes(textRecord.getLine()); - } - - private byte[] lineToBytes(String value) { - byte[] bytes = (value == null) ? Dolphin.NULL_BYTES : Dolphin.getBytes(value); - return StorageUtils.mergeByteArrays(Dolphin.getIntBytes(bytes.length), bytes); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSet.java deleted file mode 100644 index 19fd8f9dbc..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/txt/TextResultSet.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset.txt; - -import org.apache.linkis.common.io.resultset.ResultDeserializer; -import org.apache.linkis.common.io.resultset.ResultSerializer; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.StorageResultSet; - -import java.io.Serializable; - -public class TextResultSet extends StorageResultSet - implements Serializable { - - @Override - public String resultSetType() { - return ResultSetFactory.TEXT_TYPE; - } - - @Override - public ResultSerializer createResultSetSerializer() { - return new TextResultSerializer(); - } - - @Override - public ResultDeserializer createResultSetDeserializer() { - return new TextResultDeserializer(); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Compaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Compaction.java deleted file mode 100644 index abfbae9c99..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Compaction.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script; - -import org.apache.linkis.storage.script.compaction.PYScriptCompaction; -import org.apache.linkis.storage.script.compaction.QLScriptCompaction; -import org.apache.linkis.storage.script.compaction.ScalaScriptCompaction; -import org.apache.linkis.storage.script.compaction.ShellScriptCompaction; - -public interface Compaction { - String prefixConf(); - - String prefix(); - - boolean belongTo(String suffix); - - String compact(Variable variable); - - public static Compaction[] listCompactions() { - return new Compaction[] { - new PYScriptCompaction(), - new QLScriptCompaction(), - new ScalaScriptCompaction(), - new ShellScriptCompaction() - }; - } - - String getAnnotationSymbol(); -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ParserFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ParserFactory.java deleted file mode 100644 index 58edf3dfe9..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ParserFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script; - -import org.apache.linkis.storage.script.parser.PYScriptParser; -import org.apache.linkis.storage.script.parser.QLScriptParser; -import org.apache.linkis.storage.script.parser.ScalaScriptParser; -import org.apache.linkis.storage.script.parser.ShellScriptParser; - -public class ParserFactory { - public static Parser[] listParsers() { - return new Parser[] { - new PYScriptParser(), new QLScriptParser(), new ScalaScriptParser(), new ShellScriptParser() - }; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsWriter.java deleted file mode 100644 index 835e1bfb21..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsWriter.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.FsWriter; -import org.apache.linkis.storage.script.writer.StorageScriptFsWriter; - -import java.io.InputStream; -import java.io.OutputStream; - -public abstract class ScriptFsWriter extends FsWriter { - FsPath path; - String charset; - - public abstract InputStream getInputStream(); - - public static ScriptFsWriter getScriptFsWriter( - FsPath path, String charset, OutputStream outputStream) { - return new StorageScriptFsWriter(path, charset, outputStream); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Variable.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Variable.java deleted file mode 100644 index 8c726671ba..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/Variable.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script; - -public class Variable { - protected String sortParent; - protected String sort; - protected String key; - protected String value; - - public Variable(String sortParent, String sort, String key, String value) { - this.sortParent = sortParent; - this.sort = sort; - this.key = key; - this.value = value; - } - - public String getSortParent() { - return sortParent; - } - - public String getSort() { - return sort; - } - - public String getKey() { - return key; - } - - public String getValue() { - return value; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/VariableParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/VariableParser.java deleted file mode 100644 index cd10746457..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/VariableParser.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script; - -import java.util.*; - -public class VariableParser { - - public static final String CONFIGURATION = "configuration"; - public static final String VARIABLE = "variable"; - public static final String RUNTIME = "runtime"; - public static final String STARTUP = "startup"; - public static final String SPECIAL = "special"; - - public static Variable[] getVariables(Map params) { - List variables = new ArrayList<>(); - Map variableMap = - (Map) params.getOrDefault(VARIABLE, new HashMap()); - for (Map.Entry entry : variableMap.entrySet()) { - variables.add(new Variable(VARIABLE, null, entry.getKey(), entry.getValue().toString())); - } - - Map configurationMap = - (Map) params.getOrDefault(CONFIGURATION, new HashMap()); - for (Map.Entry entry : configurationMap.entrySet()) { - Map subMap = (Map) entry.getValue(); - for (Map.Entry subEntry : subMap.entrySet()) { - if (!isContextIDINFO(subEntry.getKey())) { - Object value = subEntry.getValue(); - if (value instanceof Map) { - Map innerMap = (Map) value; - for (Map.Entry innerEntry : innerMap.entrySet()) { - if (!isContextIDINFO(innerEntry.getKey())) { - variables.add( - new Variable( - entry.getKey(), - subEntry.getKey(), - innerEntry.getKey(), - innerEntry.getValue().toString())); - } - } - } else { - if (value == null) { - variables.add(new Variable(CONFIGURATION, entry.getKey(), subEntry.getKey(), "")); - } else { - variables.add( - new Variable(CONFIGURATION, entry.getKey(), subEntry.getKey(), value.toString())); - } - } - } - } - } - return variables.toArray(new Variable[variables.size()]); - } - - private static boolean isContextIDINFO(String key) { - return "contextID".equalsIgnoreCase(key) || "nodeName".equalsIgnoreCase(key); - } - - public static Map getMap(Variable[] variables) { - Map variableKey2Value = new HashMap<>(); - Map confs = new HashMap<>(); - - Arrays.stream(variables) - .filter(variable -> variable.sort == null) - .forEach(v -> variableKey2Value.put(v.key, v.value)); - - Arrays.stream(variables) - .filter(variable -> variable.sort != null) - .forEach( - v -> { - switch (v.getSort()) { - case STARTUP: - case RUNTIME: - case SPECIAL: - if (!confs.containsKey(v.getSort())) { - confs.put(v.getSort(), createMap(v)); - } else { - Map subMap = (Map) confs.get(v.getSort()); - subMap.put(v.getKey(), v.getValue()); - } - break; - default: - if (!confs.containsKey(v.getSortParent())) { - Map subMap = new HashMap<>(); - subMap.put(v.getSort(), createMap(v)); - confs.put(v.getSortParent(), subMap); - } else { - Map subMap = (Map) confs.get(v.getSortParent()); - if (!subMap.containsKey(v.getSort())) { - subMap.put(v.getSort(), createMap(v)); - } else { - Map innerMap = (Map) subMap.get(v.getSort()); - innerMap.put(v.getKey(), v.getValue()); - } - } - break; - } - }); - - Map params = new HashMap<>(); - if (!variableKey2Value.isEmpty()) { - params.put(VARIABLE, variableKey2Value); - } - if (!confs.isEmpty()) { - params.put(CONFIGURATION, confs); - } - return params; - } - - private static Map createMap(Variable variable) { - Map map = new HashMap<>(); - map.put(variable.getKey(), variable.getValue()); - return map; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/PYScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/PYScriptCompaction.java deleted file mode 100644 index 4022e7d46d..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/PYScriptCompaction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.compaction; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class PYScriptCompaction extends CommonScriptCompaction { - - private static final PYScriptCompaction pYScriptCompaction = new PYScriptCompaction(); - - public static CommonScriptCompaction apply() { - return pYScriptCompaction; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON()); - } - - @Override - public String prefix() { - return "#@set"; - } - - @Override - public String prefixConf() { - return "#conf@set"; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/QLScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/QLScriptCompaction.java deleted file mode 100644 index 97fc29efa0..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/QLScriptCompaction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.compaction; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class QLScriptCompaction extends CommonScriptCompaction { - - private static final QLScriptCompaction qLScriptCompaction = new QLScriptCompaction(); - - public static CommonScriptCompaction apply() { - return qLScriptCompaction; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL()); - } - - @Override - public String prefix() { - return "--@set"; - } - - @Override - public String prefixConf() { - return "--conf@set"; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.java deleted file mode 100644 index 18d5d2f531..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.compaction; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class ScalaScriptCompaction extends CommonScriptCompaction { - - private static final ScalaScriptCompaction compaction = new ScalaScriptCompaction(); - - public static CommonScriptCompaction apply() { - return compaction; - } - - @Override - public String prefix() { - return "//@set"; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA()); - } - - @Override - public String prefixConf() { - return "//conf@set"; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.java deleted file mode 100644 index 085815b559..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.compaction; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class ShellScriptCompaction extends CommonScriptCompaction { - - private static final ShellScriptCompaction shellScriptCompaction = new ShellScriptCompaction(); - - public static CommonScriptCompaction apply() { - return shellScriptCompaction; - } - - @Override - public String prefix() { - return "#@set"; - } - - @Override - public String prefixConf() { - return "#conf@set"; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL()); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/CommonScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/CommonScriptParser.java deleted file mode 100644 index 0d7e9f58e1..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/CommonScriptParser.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.parser; - -import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.script.Parser; -import org.apache.linkis.storage.script.Variable; -import org.apache.linkis.storage.script.VariableParser; - -import java.util.Arrays; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public abstract class CommonScriptParser implements Parser { - - @Override - public Variable parse(String line) { - String variableReg = "\\s*" + prefix() + "\\s*(.+)\\s*" + "=" + "\\s*(.+)\\s*"; - Pattern pattern = Pattern.compile(variableReg); - Matcher matcher = pattern.matcher(line); - if (matcher.matches()) { - String key = matcher.group(1).trim(); - String value = matcher.group(2).trim(); - return new Variable(VariableParser.VARIABLE, null, key, value); - - } else { - String[] splitLine = line.split("="); - if (splitLine.length != 2) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorCode(), - LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorDesc()); - } - String[] subSplit = - Arrays.stream(splitLine[0].split(" ")) - .filter(str -> !"".equals(str)) - .toArray(String[]::new); - if (subSplit.length != 4) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorCode(), - LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorDesc()); - } - if (!subSplit[0].trim().equals(prefixConf())) { - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorCode(), - LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER.getErrorDesc()); - } - String sortParent = subSplit[1].trim(); - String sort = subSplit[2].trim(); - String key = subSplit[3].trim(); - String value = splitLine[1].trim(); - return new Variable(sortParent, sort, key, value); - } - } - - @Override - public String getAnnotationSymbol() { - return prefix().split("@")[0]; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/PYScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/PYScriptParser.java deleted file mode 100644 index 3d5edcfac7..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/PYScriptParser.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.parser; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class PYScriptParser extends CommonScriptParser { - - private static final PYScriptParser pYScriptParser = new PYScriptParser(); - - public static CommonScriptParser apply() { - return pYScriptParser; - } - - @Override - public String prefix() { - return "#@set"; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON()); - } - - @Override - public String prefixConf() { - return "#conf@set"; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/QLScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/QLScriptParser.java deleted file mode 100644 index 806a84760f..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/QLScriptParser.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.parser; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class QLScriptParser extends CommonScriptParser { - - private static final QLScriptParser qLScriptParser = new QLScriptParser(); - - public static CommonScriptParser apply() { - return qLScriptParser; - } - - @Override - public String prefix() { - return "--@set"; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL()); - } - - @Override - public String prefixConf() { - return "--conf@set"; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ScalaScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ScalaScriptParser.java deleted file mode 100644 index 982538d8e8..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ScalaScriptParser.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.parser; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class ScalaScriptParser extends CommonScriptParser { - - private static final ScalaScriptParser scalaScriptParser = new ScalaScriptParser(); - - public static CommonScriptParser apply() { - return scalaScriptParser; - } - - @Override - public String prefix() { - return "//@set"; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA()); - } - - @Override - public String prefixConf() { - return "//conf@set"; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ShellScriptParser.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ShellScriptParser.java deleted file mode 100644 index e1eebe3e58..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/parser/ShellScriptParser.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.parser; - -import org.apache.linkis.common.utils.CodeAndRunTypeUtils; - -public class ShellScriptParser extends CommonScriptParser { - - private static final ShellScriptParser shellScriptParser = new ShellScriptParser(); - - public static CommonScriptParser create() { - return shellScriptParser; - } - - @Override - public String prefix() { - return "#@set"; - } - - @Override - public boolean belongTo(String suffix) { - return CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( - suffix, CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL()); - } - - @Override - public String prefixConf() { - return "#conf@set"; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/reader/StorageScriptFsReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/reader/StorageScriptFsReader.java deleted file mode 100644 index 1f862fa6b1..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/reader/StorageScriptFsReader.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.reader; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.storage.script.*; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.io.IOUtils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class StorageScriptFsReader extends ScriptFsReader { - private final FsPath path; - private final String charset; - private final InputStream inputStream; - - private InputStreamReader inputStreamReader; - private BufferedReader bufferedReader; - - private ScriptMetaData metadata; - - private List variables = new ArrayList<>(); - private String lineText; - - public StorageScriptFsReader(FsPath path, String charset, InputStream inputStream) { - super(path, charset); - this.path = path; - this.charset = charset; - this.inputStream = inputStream; - } - - @Override - public Record getRecord() throws IOException { - if (metadata == null) throw new IOException("Must read metadata first(必须先读取metadata)"); - ScriptRecord record = new ScriptRecord(lineText); - lineText = bufferedReader.readLine(); - return record; - } - - @Override - public MetaData getMetaData() throws IOException { - if (metadata == null) init(); - Parser parser = getScriptParser(); - lineText = bufferedReader.readLine(); - while (hasNext() - && Objects.nonNull(parser) - && isMetadata(lineText, parser.prefix(), parser.prefixConf())) { - variables.add(parser.parse(lineText)); - lineText = bufferedReader.readLine(); - } - metadata = new ScriptMetaData(variables.toArray(new Variable[0])); - return metadata; - } - - public void init() { - inputStreamReader = new InputStreamReader(inputStream); - bufferedReader = new BufferedReader(inputStreamReader); - } - - @Override - public int skip(int recordNum) throws IOException { - if (recordNum < 0) return -1; - if (metadata == null) getMetaData(); - try { - return (int) bufferedReader.skip(recordNum); - } catch (Throwable t) { - return recordNum; - } - } - - @Override - public long getPosition() throws IOException { - return -1L; - } - - @Override - public boolean hasNext() throws IOException { - return lineText != null; - } - - @Override - public long available() throws IOException { - return inputStream != null ? inputStream.available() : 0L; - } - - @Override - public void close() throws IOException { - IOUtils.closeQuietly(bufferedReader); - IOUtils.closeQuietly(inputStreamReader); - IOUtils.closeQuietly(inputStream); - } - - /** - * Determine if the read line is metadata(判断读的行是否是metadata) - * - * @param line - * @return - */ - public boolean isMetadata(String line, String prefix, String prefixConf) { - String regex = "\\s*" + prefix + "\\s*(.+)\\s*=\\s*(.+)\\s*"; - if (line.matches(regex)) { - return true; - } else { - String[] split = line.split("="); - if (split.length != 2) { - return false; - } - if (Stream.of(split[0].split(" ")).filter(str -> !"".equals(str)).count() != 4) { - return false; - } - - Optional optional = - Stream.of(split[0].split(" ")).filter(str -> !"".equals(str)).findFirst(); - if (optional.isPresent() && !optional.get().equals(prefixConf)) { - return false; - } - return true; - } - } - - /** - * get the script parser according to the path(根据文件路径 获取对应的script parser ) - * - * @return Scripts Parser - */ - public Parser getScriptParser() { - List parsers = - Arrays.stream(ParserFactory.listParsers()) - .filter(p -> p.belongTo(StorageUtils.pathToSuffix(path.getPath()))) - .collect(Collectors.toList()); - if (parsers.size() > 0) { - return parsers.get(0); - } else { - return null; - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.java deleted file mode 100644 index 84dd6abb83..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.script.writer; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.script.Compaction; -import org.apache.linkis.storage.script.ScriptFsWriter; -import org.apache.linkis.storage.script.ScriptMetaData; -import org.apache.linkis.storage.script.Variable; -import org.apache.linkis.storage.utils.StorageConfiguration; -import org.apache.linkis.storage.utils.StorageUtils; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class StorageScriptFsWriter extends ScriptFsWriter { - private static final Logger log = LoggerFactory.getLogger(StorageScriptFsWriter.class); - - private final FsPath path; - private final String charset; - private final OutputStream outputStream; - private final StringBuilder stringBuilder = new StringBuilder(); - - public StorageScriptFsWriter(FsPath path, String charset, OutputStream outputStream) { - this.path = path; - this.charset = charset; - this.outputStream = outputStream; - } - - @Override - public void addMetaData(MetaData metaData) throws IOException { - String suffix = StorageUtils.pathToSuffix(path.getPath()); - List compactions = - Stream.of(Compaction.listCompactions()) - .filter(compaction -> compaction.belongTo(suffix)) - .collect(Collectors.toList()); - List metadataLine = new ArrayList<>(); - if (!compactions.isEmpty()) { - Variable[] metaData1 = ((ScriptMetaData) metaData).getMetaData(); - Stream.of(metaData1).map(compactions.get(0)::compact).forEach(metadataLine::add); - - // add annotition symbol - if (metadataLine.size() > 0) { - metadataLine.add(compactions.get(0).getAnnotationSymbol()); - } - if (outputStream != null) { - IOUtils.writeLines(metadataLine, "\n", outputStream, charset); - } else { - metadataLine.forEach(m -> stringBuilder.append(m).append("\n")); - } - } - } - - @Override - public void addRecord(Record record) throws IOException { - LineRecord scriptRecord = (LineRecord) record; - if (outputStream != null) { - IOUtils.write(scriptRecord.getLine(), outputStream, charset); - } else { - stringBuilder.append(scriptRecord.getLine()); - } - } - - @Override - public void close() { - IOUtils.closeQuietly(outputStream); - } - - @Override - public void flush() { - if (outputStream instanceof HdfsDataOutputStream) { - try { - ((HdfsDataOutputStream) outputStream).hflush(); - } catch (IOException t) { - log.warn("Error encountered when flush script", t); - } - } else if (outputStream != null) { - try { - outputStream.flush(); - } catch (IOException t) { - log.warn("Error encountered when flush script", t); - } - } - } - - @Override - public InputStream getInputStream() { - byte[] bytes = null; - try { - bytes = - stringBuilder.toString().getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue()); - } catch (UnsupportedEncodingException e) { - log.warn("StorageScriptFsWriter getInputStream failed", e); - } - return new ByteArrayInputStream(bytes); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/AbstractFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/AbstractFileSource.java deleted file mode 100644 index fc4e615b36..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/AbstractFileSource.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.source; - -import org.apache.linkis.common.io.FsWriter; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.math3.util.Pair; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -public abstract class AbstractFileSource implements FileSource { - - private FileSplit[] fileSplits; - - public AbstractFileSource(FileSplit[] fileSplits) { - this.fileSplits = fileSplits; - } - - @Override - public FileSource shuffle(Function function) { - Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.shuffler = function); - return this; - } - - @Override - public FileSource page(int page, int pageSize) { - Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.page(page, pageSize)); - return this; - } - - @Override - public FileSource addParams(Map params) { - Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.addParams(params)); - return this; - } - - @Override - public FileSource addParams(String key, String value) { - Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.addParams(key, value)); - return this; - } - - @Override - public FileSplit[] getFileSplits() { - return this.fileSplits; - } - - @Override - public Map getParams() { - return Arrays.stream(fileSplits) - .map(FileSplit::getParams) - .flatMap(map -> map.entrySet().stream()) - .collect( - Collectors.toMap( - Map.Entry::getKey, Map.Entry::getValue, (existingValue, newValue) -> newValue)); - } - - @Override - public void write(FsWriter fsWriter) { - Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.write(fsWriter)); - } - - @Override - public void close() { - Arrays.stream(fileSplits).forEach(IOUtils::closeQuietly); - } - - @Override - public Pair>[] collect() { - return Arrays.stream(fileSplits).map(FileSplit::collect).toArray(Pair[]::new); - } - - @Override - public int getTotalLine() { - return Arrays.stream(fileSplits).mapToInt(FileSplit::getTotalLine).sum(); - } - - @Override - public String[] getTypes() { - return Arrays.stream(fileSplits).map(FileSplit::getType).toArray(String[]::new); - } - - @Override - public Pair[] getFileInfo(int needToCountRowNumber) { - return Arrays.stream(fileSplits) - .map(fileSplit -> fileSplit.getFileInfo(needToCountRowNumber)) - .toArray(Pair[]::new); - } - - @Override - public FileSource limitBytes(Long limitBytes) { - Arrays.stream(fileSplits).forEach(fileSplit -> fileSplit.setLimitBytes(limitBytes)); - return this; - } - - @Override - public FileSource limitColumnLength(int limitColumnLength) { - Arrays.stream(fileSplits) - .forEach(fileSplit -> fileSplit.setLimitColumnLength(limitColumnLength)); - return this; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java deleted file mode 100644 index f7a5d3e512..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.source; - -import org.apache.linkis.common.io.*; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.storage.conf.LinkisStorageConf; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.ResultSetReaderFactory; -import org.apache.linkis.storage.script.ScriptFsReader; -import org.apache.linkis.storage.utils.StorageConfiguration; - -import org.apache.commons.math3.util.Pair; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.function.BiFunction; -import java.util.function.Function; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE; - -public interface FileSource extends Closeable { - Logger logger = LoggerFactory.getLogger(FileSource.class); - - FileSource shuffle(Function s); - - FileSource page(int page, int pageSize); - - Pair>[] collect(); - - Pair[] getFileInfo(int needToCountRowNumber); - - void write(FsWriter fsWriter); - - FileSource addParams(Map params); - - FileSource addParams(String key, String value); - - Map getParams(); - - int getTotalLine(); - - String[] getTypes(); - - FileSplit[] getFileSplits(); - - String[] fileType = LinkisStorageConf.getFileTypeArr(); - BiFunction suffixPredicate = - (path, suffix) -> path.endsWith("." + suffix); - - static boolean isResultSet(String path) { - return suffixPredicate.apply(path, LinkisStorageConf.DOLPHIN) - || suffixPredicate.apply(path, LinkisStorageConf.PARQUET) - || suffixPredicate.apply(path, LinkisStorageConf.ORC); - } - - static boolean isResultSet(FsPath fsPath) { - return isResultSet(fsPath.getPath()); - } - - FileSource limitBytes(Long limitBytes); - - FileSource limitColumnLength(int limitColumnLength); - - /** - * Currently only supports table multi-result sets - * - * @param fsPaths - * @param fs - * @return - */ - static FileSource create(FsPath[] fsPaths, Fs fs) { - // Filter non-table result sets - FileSplit[] fileSplits = - Arrays.stream(fsPaths) - .map(fsPath -> createResultSetFileSplit(fsPath, fs)) - .filter(FileSource::isTableResultSet) - .toArray(FileSplit[]::new); - return new ResultsetFileSource(fileSplits); - } - - static boolean isTableResultSet(FileSplit fileSplit) { - return fileSplit.type.equals(ResultSetFactory.TABLE_TYPE); - } - - static boolean isTableResultSet(FileSource fileSource) { - // Return true only if all splits are table result sets - return Arrays.stream(fileSource.getFileSplits()).allMatch(FileSource::isTableResultSet); - } - - static FileSource create(FsPath fsPath, Fs fs) { - if (!canRead(fsPath.getPath())) { - throw new StorageWarnException( - UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), UNSUPPORTED_OPEN_FILE_TYPE.getErrorDesc()); - } - if (isResultSet(fsPath)) { - return new ResultsetFileSource(new FileSplit[] {createResultSetFileSplit(fsPath, fs)}); - } else { - return new TextFileSource(new FileSplit[] {createTextFileSplit(fsPath, fs)}); - } - } - - static FileSource create(FsPath fsPath, InputStream is) { - if (!canRead(fsPath.getPath())) { - throw new StorageWarnException( - UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), UNSUPPORTED_OPEN_FILE_TYPE.getErrorDesc()); - } - if (isResultSet(fsPath)) { - return new ResultsetFileSource(new FileSplit[] {createResultSetFileSplit(fsPath, is)}); - } else { - return new TextFileSource(new FileSplit[] {createTextFileSplit(fsPath, is)}); - } - } - - static FileSplit createResultSetFileSplit(FsPath fsPath, InputStream is) { - logger.info("try create result set file split with path:{}", fsPath.getPath()); - ResultSet resultset = ResultSetFactory.getInstance().getResultSetByPath(fsPath); - ResultSetReader resultsetReader = - ResultSetReaderFactory.getResultSetReader(resultset, is, fsPath); - return new FileSplit(resultsetReader, resultset.resultSetType()); - } - - static FileSplit createResultSetFileSplit(FsPath fsPath, Fs fs) { - ResultSet resultset = ResultSetFactory.getInstance().getResultSetByPath(fsPath, fs); - ResultSetReader resultsetReader = null; - try { - resultsetReader = - ResultSetReaderFactory.getResultSetReader(resultset, fs.read(fsPath), fsPath); - } catch (IOException e) { - logger.warn("FileSource createResultSetFileSplit failed", e); - } - return new FileSplit(resultsetReader, resultset.resultSetType()); - } - - static FileSplit createTextFileSplit(FsPath fsPath, InputStream is) { - ScriptFsReader scriptFsReader = - ScriptFsReader.getScriptFsReader( - fsPath, StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue(), is); - return new FileSplit(scriptFsReader); - } - - static FileSplit createTextFileSplit(FsPath fsPath, Fs fs) { - ScriptFsReader scriptFsReader = null; - try { - scriptFsReader = - ScriptFsReader.getScriptFsReader( - fsPath, StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue(), fs.read(fsPath)); - } catch (IOException e) { - logger.warn("FileSource createTextFileSplit failed", e); - } - return new FileSplit(scriptFsReader); - } - - static boolean canRead(String path) { - return Arrays.stream(fileType).anyMatch(suffix -> path.endsWith("." + suffix)); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSplit.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSplit.java deleted file mode 100644 index 3a6c05a54a..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSplit.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.source; - -import org.apache.linkis.common.io.FsReader; -import org.apache.linkis.common.io.FsWriter; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.domain.DataType; -import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; -import org.apache.linkis.storage.script.Parser; -import org.apache.linkis.storage.script.ScriptMetaData; -import org.apache.linkis.storage.script.Variable; -import org.apache.linkis.storage.script.VariableParser; -import org.apache.linkis.storage.script.reader.StorageScriptFsReader; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.math3.util.Pair; - -import java.io.Closeable; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FileSplit implements Closeable { - private static final Logger logger = LoggerFactory.getLogger(FileSplit.class); - - private FsReader fsReader; - protected String type = "script/text"; - private int start = 0; - private int end = -1; - private int count = 0; - private int totalLine = 0; - protected Function shuffler; - private boolean pageTrigger = false; - protected Map params = new HashMap<>(); - private long limitBytes = 0L; - private int limitColumnLength = 0; - - public FileSplit(FsReader fsReader) { - this.fsReader = fsReader; - } - - public FileSplit(FsReader fsReader, String type) { - this.fsReader = fsReader; - this.type = type; - } - - public void page(int page, int pageSize) { - if (!pageTrigger) { - start = (page - 1) * pageSize; - end = pageSize * page - 1; - pageTrigger = true; - } - } - - public String getType() { - return type; - } - - public void addParams(Map params) { - this.params.putAll(params); - } - - public void addParams(String key, String value) { - this.params.put(key, value); - } - - public Map getParams() { - return params; - } - - public int getTotalLine() { - return totalLine; - } - - public void setLimitBytes(long limitBytes) { - this.limitBytes = limitBytes; - } - - public void setLimitColumnLength(int limitColumnLength) { - this.limitColumnLength = limitColumnLength; - } - - public M whileLoop(Function metaDataFunction, Consumer recordConsumer) { - M m = null; - try { - MetaData metaData = fsReader.getMetaData(); - m = metaDataFunction.apply(metaData); - if (pageTrigger) { - fsReader.skip(start); - } - count = start; - boolean hasRemovedFlag = false; - while (fsReader.hasNext() && ifContinueRead()) { - Record record = fsReader.getRecord(); - boolean needRemoveFlag = false; - if (!hasRemovedFlag && fsReader instanceof StorageScriptFsReader) { - Parser parser = ((StorageScriptFsReader) fsReader).getScriptParser(); - Variable[] meta = ((ScriptMetaData) metaData).getMetaData(); - if (meta != null - && meta.length > 0 - && parser != null - && parser.getAnnotationSymbol().equals(record.toString())) { - needRemoveFlag = true; - hasRemovedFlag = true; - } - } - if (!needRemoveFlag) { - recordConsumer.accept(shuffler.apply(record)); - totalLine++; - count++; - } - } - } catch (IOException e) { - logger.warn("FileSplit forEach failed", e); - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage()); - } - return m; - } - - public void biConsumerWhileLoop( - Consumer metaDataFunction, Consumer recordConsumer) { - try { - MetaData metaData = fsReader.getMetaData(); - metaDataFunction.accept(metaData); - if (pageTrigger) { - fsReader.skip(start); - } - count = start; - boolean hasRemovedFlag = false; - while (fsReader.hasNext() && ifContinueRead()) { - Record record = fsReader.getRecord(); - boolean needRemoveFlag = false; - if (!hasRemovedFlag && fsReader instanceof StorageScriptFsReader) { - Parser parser = ((StorageScriptFsReader) fsReader).getScriptParser(); - Variable[] meta = ((ScriptMetaData) metaData).getMetaData(); - if (meta != null - && meta.length > 0 - && parser != null - && parser.getAnnotationSymbol().equals(record.toString())) { - needRemoveFlag = true; - hasRemovedFlag = true; - } - } - if (!needRemoveFlag) { - recordConsumer.accept(shuffler.apply(record)); - totalLine++; - count++; - } - } - } catch (IOException e) { - logger.warn("FileSplit forEach failed", e); - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage()); - } - } - - public Pair getFileInfo(int needToCountRowNumber) { - int colNumber = 0; - int rowNumber = 0; - MetaData metaData = null; - try { - metaData = fsReader.getMetaData(); - colNumber = - metaData instanceof TableMetaData ? ((TableMetaData) metaData).getColumns().length : 1; - rowNumber = - needToCountRowNumber == -1 - ? fsReader.skip(Integer.MAX_VALUE) - : fsReader.skip(needToCountRowNumber); - } catch (IOException e) { - logger.warn("FileSplit getFileInfo failed", e); - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage()); - } - return new Pair<>(colNumber, rowNumber); - } - - public void write(FsWriter fsWriter) { - biConsumerWhileLoop( - metaData -> { - try { - fsWriter.addMetaData(metaData); - } catch (IOException e) { - logger.warn("FileSplit addMetaData failed", e); - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage()); - } - }, - record -> { - try { - fsWriter.addRecord(record); - } catch (IOException e) { - logger.warn("FileSplit addRecord failed", e); - throw new StorageWarnException( - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode(), - LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE.getErrorMessage()); - } - }); - } - - public Pair> collect() { - List recordList = new ArrayList<>(); - final AtomicLong tmpBytes = new AtomicLong(0L); - final AtomicBoolean overFlag = new AtomicBoolean(false); - Object metaData = - whileLoop( - collectMetaData -> collectMetaData(collectMetaData), - r -> { - if (!overFlag.get()) { - String[] arr = collectRecord(r); - if (limitBytes > 0) { - for (int i = 0; i < arr.length; i++) { - tmpBytes.addAndGet(arr[i].getBytes().length); - if (overFlag.get() || tmpBytes.get() > limitBytes) { - overFlag.set(true); - arr[i] = ""; - } - } - recordList.add(arr); - } else { - recordList.add(arr); - } - } - }); - return new Pair<>(metaData, recordList); - } - - public String[] collectRecord(Record record) { - if (record instanceof TableRecord) { - TableRecord tableRecord = (TableRecord) record; - if (limitColumnLength > 0) { - return Arrays.stream(tableRecord.row) - .map( - obj -> { - String col = DataType.valueToString(obj); - if (col.length() > limitColumnLength) { - return col.substring(0, limitColumnLength); - } else { - return col; - } - }) - .toArray(String[]::new); - } - return Arrays.stream(tableRecord.row).map(DataType::valueToString).toArray(String[]::new); - } else if (record instanceof LineRecord) { - LineRecord lineRecord = (LineRecord) record; - return new String[] {lineRecord.getLine()}; - } else { - throw new IllegalArgumentException("Unknown record type"); - } - } - - public Object collectMetaData(MetaData metaData) { - if (metaData instanceof ScriptMetaData) { - ScriptMetaData scriptMetaData = (ScriptMetaData) metaData; - return VariableParser.getMap(scriptMetaData.getMetaData()); - } else if (metaData instanceof LineMetaData) { - LineMetaData lineMetaData = (LineMetaData) metaData; - return lineMetaData.getMetaData(); - } else if (metaData instanceof TableMetaData) { - TableMetaData tableMetaData = (TableMetaData) metaData; - return Arrays.stream(tableMetaData.getColumns()) - .map(this::columnToMap) - .collect(Collectors.toList()); - } else { - throw new IllegalArgumentException("Unknown metadata type"); - } - } - - private Map columnToMap(Column column) { - Map stringMap = new HashMap<>(); - stringMap.put("columnName", column.getColumnName()); - stringMap.put("comment", column.getComment()); - stringMap.put("dataType", column.getDataType().getTypeName()); - return stringMap; - } - - public boolean ifContinueRead() { - return !pageTrigger || count <= end; - } - - public boolean ifStartRead() { - return !pageTrigger || count >= start; - } - - @Override - public void close() { - IOUtils.closeQuietly(fsReader); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java deleted file mode 100644 index 54fd64daad..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.source; - -import org.apache.linkis.storage.domain.Dolphin; -import org.apache.linkis.storage.resultset.table.TableRecord; -import org.apache.linkis.storage.utils.StorageUtils; - -import java.util.Arrays; - -public class ResultsetFileSource extends AbstractFileSource { - - public ResultsetFileSource(FileSplit[] fileSplits) { - super(fileSplits); - shuffle( - record -> { - if (record instanceof TableRecord) { - TableRecord tableRecord = (TableRecord) record; - String nullValue = getParams().getOrDefault("nullValue", "NULL"); - return new TableRecord( - Arrays.stream(tableRecord.row) - .map( - r -> { - if (r == null || r.equals("NULL")) { - if (nullValue.equals(Dolphin.LINKIS_NULL)) { - return r; - } else { - return nullValue; - } - } else if (r.equals("")) { - String emptyValue = getParams().getOrDefault("nullValue", ""); - if (emptyValue.equals(Dolphin.LINKIS_NULL)) { - return ""; - } else { - return emptyValue; - } - } else if (r instanceof Double) { - return StorageUtils.doubleToString((Double) r); - } else { - return r; - } - }) - .toArray()); - } else { - return record; - } - }); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/TextFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/TextFileSource.java deleted file mode 100644 index 7e5396bf74..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/TextFileSource.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.source; - -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.script.ScriptRecord; - -import org.apache.commons.math3.util.Pair; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -class TextFileSource extends AbstractFileSource { - public static final String[] LINE_BREAKER = new String[] {"\n"}; - - public TextFileSource(FileSplit[] fileSplits) { - super(fileSplits); - shuffle( - record -> { - if (record instanceof ScriptRecord && "".equals(((ScriptRecord) record).getLine())) { - return new LineRecord("\n"); - } else { - return record; - } - }); - } - - @Override - public Pair>[] collect() { - Pair>[] collects = super.collect(); - if (!getParams().getOrDefault("ifMerge", "true").equals("true")) { - return collects; - } - ArrayList> snds = - Arrays.stream(collects) - .map(Pair::getSecond) - .collect(Collectors.toCollection(ArrayList::new)); - snds.forEach( - snd -> { - StringBuilder str = new StringBuilder(); - snd.forEach( - arr -> { - if (Arrays.equals(arr, LINE_BREAKER)) { - str.append("\n"); - } else { - str.append(arr[0]).append("\n"); - } - }); - snd.clear(); - snd.add(new String[] {str.toString()}); - }); - return collects; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java deleted file mode 100644 index 4c50479637..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.utils; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.storage.FSFactory; -import org.apache.linkis.storage.fs.FileSystem; -import org.apache.linkis.storage.fs.impl.LocalFileSystem; - -import org.apache.commons.io.IOUtils; - -import java.io.IOException; -import java.util.Objects; -import java.util.Stack; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FileSystemUtils { - private static final Logger logger = LoggerFactory.getLogger(FileSystemUtils.class); - - public static void copyFile(FsPath filePath, FsPath origin, String user) throws Exception { - FileSystem fileSystem = (FileSystem) FSFactory.getFsByProxyUser(filePath, user); - try { - fileSystem.init(null); - if (!fileSystem.exists(filePath)) { - if (!fileSystem.exists(filePath.getParent())) { - fileSystem.mkdirs(filePath.getParent()); - } - fileSystem.createNewFile(filePath); - } - fileSystem.copyFile(origin, filePath); - } finally { - IOUtils.closeQuietly(fileSystem); - } - } - - /** - * Create a new file - * - * @param filePath - * @param createParentWhenNotExists Whether to recursively create a directory - */ - public static void createNewFile(FsPath filePath, boolean createParentWhenNotExists) - throws Exception { - createNewFile(filePath, StorageUtils.getJvmUser(), createParentWhenNotExists); - } - - public static void createNewFile(FsPath filePath, String user, boolean createParentWhenNotExists) - throws Exception { - FileSystem fileSystem = (FileSystem) FSFactory.getFsByProxyUser(filePath, user); - try { - fileSystem.init(null); - createNewFileWithFileSystem(fileSystem, filePath, user, createParentWhenNotExists); - } finally { - IOUtils.closeQuietly(fileSystem); - } - } - - public static void createNewFileWithFileSystem( - FileSystem fileSystem, FsPath filePath, String user, boolean createParentWhenNotExists) - throws Exception { - if (!fileSystem.exists(filePath)) { - if (!fileSystem.exists(filePath.getParent())) { - if (!createParentWhenNotExists) { - throw new IOException( - "parent dir " + filePath.getParent().getPath() + " dose not exists."); - } - mkdirs(fileSystem, filePath.getParent(), user); - } - fileSystem.createNewFile(filePath); - if (fileSystem instanceof LocalFileSystem) { - fileSystem.setOwner(filePath, user); - } else { - logger.info("doesn't need to call setOwner"); - } - } - } - - /** - * create new file and set file owner by FileSystem - * - * @param fileSystem - * @param filePath - * @param user - * @param createParentWhenNotExists - */ - public static void createNewFileAndSetOwnerWithFileSystem( - FileSystem fileSystem, FsPath filePath, String user, boolean createParentWhenNotExists) - throws Exception { - if (!fileSystem.exists(filePath)) { - if (!fileSystem.exists(filePath.getParent())) { - if (!createParentWhenNotExists) { - throw new IOException( - "parent dir " + filePath.getParent().getPath() + " dose not exists."); - } - mkdirs(fileSystem, filePath.getParent(), user); - } - fileSystem.createNewFile(filePath); - fileSystem.setOwner(filePath, user); - } - } - - /** - * Recursively create a directory - * - * @param fileSystem - * @param dest - * @param user - * @throws IOException - * @return - */ - public static boolean mkdirs(FileSystem fileSystem, FsPath dest, String user) throws IOException { - FsPath parentPath = dest.getParent(); - Stack dirsToMake = new Stack<>(); - dirsToMake.push(dest); - while (!fileSystem.exists(parentPath)) { - dirsToMake.push(parentPath); - - if (Objects.isNull(parentPath.getParent())) { - // parent path of root is null - break; - } - - parentPath = parentPath.getParent(); - } - if (!fileSystem.canExecute(parentPath)) { - throw new IOException("You have not permission to access path " + dest.getPath()); - } - while (!dirsToMake.empty()) { - FsPath path = dirsToMake.pop(); - fileSystem.mkdir(path); - if (fileSystem instanceof LocalFileSystem) { - fileSystem.setOwner(path, user); - } else { - logger.info("doesn't need to call setOwner"); - } - } - return true; - } - - /** - * Recursively create a directory(递归创建目录) add owner info - * - * @param fileSystem - * @param dest - * @param user - * @throws IOException - * @return - */ - public static boolean mkdirsAndSetOwner(FileSystem fileSystem, FsPath dest, String user) - throws IOException { - FsPath parentPath = dest.getParent(); - Stack dirsToMake = new Stack<>(); - dirsToMake.push(dest); - while (!fileSystem.exists(parentPath)) { - dirsToMake.push(parentPath); - - if (Objects.isNull(parentPath.getParent())) { - // parent path of root is null - break; - } - - parentPath = parentPath.getParent(); - } - if (!fileSystem.canExecute(parentPath)) { - throw new IOException("You have not permission to access path " + dest.getPath()); - } - while (!dirsToMake.empty()) { - FsPath path = dirsToMake.pop(); - fileSystem.mkdir(path); - fileSystem.setOwner(path, user); - } - return true; - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/OrcUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/OrcUtils.java deleted file mode 100644 index 72c2f8c355..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/OrcUtils.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.utils; - -import org.apache.linkis.storage.domain.DataType; - -import org.apache.orc.TypeDescription; -import org.apache.orc.storage.common.type.HiveDecimal; -import org.apache.orc.storage.ql.exec.vector.*; - -import java.math.BigDecimal; -import java.sql.Timestamp; -import java.text.ParseException; -import java.text.SimpleDateFormat; - -/** - * Inspired by: - * https://github.com/apache/flink/blob/master/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java - */ -public class OrcUtils { - - public static TypeDescription dataTypeToOrcType(DataType type) { - switch (type) { - case CharType: - return TypeDescription.createChar().withMaxLength(1024); - case StringType: - return TypeDescription.createString(); - case LongType: - return TypeDescription.createLong(); - case VarcharType: - return TypeDescription.createVarchar().withMaxLength(1024); - case BooleanType: - return TypeDescription.createBoolean(); - case BinaryType: - return TypeDescription.createBinary(); - case DecimalType: - return TypeDescription.createDecimal().withScale(10).withPrecision(38); - case TinyIntType: - return TypeDescription.createByte(); - case ShortIntType: - return TypeDescription.createShort(); - case IntType: - return TypeDescription.createInt(); - case BigIntType: - return TypeDescription.createLong(); - case FloatType: - return TypeDescription.createFloat(); - case DoubleType: - return TypeDescription.createDouble(); - case DateType: - return TypeDescription.createDate(); - case TimestampType: - return TypeDescription.createTimestamp(); - case ArrayType: - return TypeDescription.createList(dataTypeToOrcType(DataType.VarcharType)); - case MapType: - return TypeDescription.createMap( - dataTypeToOrcType(DataType.VarcharType), dataTypeToOrcType(DataType.VarcharType)); - default: - throw new UnsupportedOperationException("Unsupported type: " + type); - } - } - - public static void setColumn(int columnId, ColumnVector column, DataType type, Object value) { - switch (type) { - case CharType: - case VarcharType: - case BinaryType: - case StringType: - { - BytesColumnVector vector = (BytesColumnVector) column; - vector.setVal(columnId, String.valueOf(value).getBytes()); - break; - } - case BooleanType: - { - LongColumnVector vector = (LongColumnVector) column; - vector.vector[columnId] = Boolean.valueOf(value.toString()) ? 1 : 0; - break; - } - case DecimalType: - { - DecimalColumnVector vector = (DecimalColumnVector) column; - vector.set(columnId, HiveDecimal.create(new BigDecimal(value.toString()))); - break; - } - case TinyIntType: - { - LongColumnVector vector = (LongColumnVector) column; - vector.vector[columnId] = (byte) value; - break; - } - case DateType: - case IntType: - { - LongColumnVector vector = (LongColumnVector) column; - vector.vector[columnId] = Integer.valueOf(value.toString()); - break; - } - case BigIntType: - { - LongColumnVector vector = (LongColumnVector) column; - vector.vector[columnId] = Long.valueOf(value.toString()); - break; - } - case FloatType: - { - DoubleColumnVector vector = (DoubleColumnVector) column; - vector.vector[columnId] = Float.valueOf(value.toString()); - break; - } - case DoubleType: - { - DoubleColumnVector vector = (DoubleColumnVector) column; - vector.vector[columnId] = Double.valueOf(value.toString()); - break; - } - case TimestampType: - { - TimestampColumnVector vector = (TimestampColumnVector) column; - try { - vector.set( - columnId, - new Timestamp( - new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ") - .parse(value.toString()) - .getTime())); - } catch (ParseException e) { - vector.set(columnId, new Timestamp(System.currentTimeMillis())); - } - break; - } - default: - throw new UnsupportedOperationException("Unsupported type: " + type); - } - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageConfiguration.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageConfiguration.java deleted file mode 100644 index 70a3839b62..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageConfiguration.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.utils; - -import org.apache.linkis.common.conf.ByteType; -import org.apache.linkis.common.conf.CommonVars; - -import java.util.List; - -import com.google.common.collect.Lists; - -public class StorageConfiguration { - - public static CommonVars PROXY_USER = - new CommonVars<>("wds.linkis.storage.proxy.user", "${UM}", null, null); - - public static CommonVars STORAGE_ROOT_USER = - new CommonVars<>("wds.linkis.storage.root.user", "hadoop", null, null); - - public static CommonVars HDFS_ROOT_USER = - new CommonVars<>("wds.linkis.storage.hdfs.root.user", "hadoop", null, null); - - public static CommonVars LOCAL_ROOT_USER = - new CommonVars<>("wds.linkis.storage.local.root.user", "root", null, null); - - public static CommonVars STORAGE_USER_GROUP = - new CommonVars<>("wds.linkis.storage.fileSystem.group", "bdap", null, null); - - public static CommonVars STORAGE_RS_FILE_TYPE = - new CommonVars<>("wds.linkis.storage.rs.file.type", "utf-8", null, null); - - public static CommonVars STORAGE_RS_FILE_SUFFIX = - new CommonVars<>("wds.linkis.storage.rs.file.suffix", ".dolphin", null, null); - - public static CommonVars LINKIS_STORAGE_FS_LABEL = - new CommonVars<>("linkis.storage.default.fs.label", "linkis-storage", null, null); - - public static List ResultTypes = - Lists.newArrayList("%TEXT", "%TABLE", "%HTML", "%IMG", "%ANGULAR", "%SVG"); - - public static CommonVars STORAGE_RESULT_SET_PACKAGE = - new CommonVars<>( - "wds.linkis.storage.result.set.package", - "org.apache.linkis.storage.resultset", - null, - null); - - public static CommonVars STORAGE_RESULT_SET_CLASSES = - new CommonVars<>( - "wds.linkis.storage.result.set.classes", - "txt.TextResultSet,table.TableResultSet,io.IOResultSet,html.HtmlResultSet,picture.PictureResultSet", - null, - null); - - public static CommonVars STORAGE_BUILD_FS_CLASSES = - new CommonVars<>( - "wds.linkis.storage.build.fs.classes", - "org.apache.linkis.storage.factory.impl.BuildHDFSFileSystem,org.apache.linkis.storage.factory.impl.BuildLocalFileSystem," - + "org.apache.linkis.storage.factory.impl.BuildOSSSystem,org.apache.linkis.storage.factory.impl.BuildS3FileSystem", - null, - null); - - public static CommonVars IS_SHARE_NODE = - new CommonVars<>("wds.linkis.storage.is.share.node", true, null, null); - - public static CommonVars ENABLE_IO_PROXY = - new CommonVars<>("wds.linkis.storage.enable.io.proxy", false, null, null); - - public static CommonVars IO_USER = - new CommonVars<>("wds.linkis.storage.io.user", "root", null, null); - public static CommonVars IO_FS_EXPIRE_TIME = - new CommonVars<>("wds.linkis.storage.io.fs.num", 1000 * 60 * 10, null, null); - - public static CommonVars IO_PROXY_READ_FETCH_SIZE = - new CommonVars<>("wds.linkis.storage.io.read.fetch.size", new ByteType("100k"), null, null); - - public static CommonVars IO_PROXY_WRITE_CACHE_SIZE = - new CommonVars<>("wds.linkis.storage.io.write.cache.size", new ByteType("64k"), null, null); - - public static CommonVars IO_DEFAULT_CREATOR = - new CommonVars<>("wds.linkis.storage.io.default.creator", "IDE", null, null); - public static CommonVars IO_FS_RE_INIT = - new CommonVars<>("wds.linkis.storage.io.fs.re.init", "re-init", null, null); - - public static CommonVars IO_INIT_RETRY_LIMIT = - new CommonVars<>("wds.linkis.storage.io.init.retry.limit", 10, null, null); - - public static CommonVars STORAGE_HDFS_GROUP = - new CommonVars<>("wds.linkis.storage.fileSystem.hdfs.group", "hadoop", null, null); - - public static CommonVars DOUBLE_FRACTION_LEN = - new CommonVars<>("wds.linkis.double.fraction.length", 30, null, null); - - public static CommonVars HDFS_PATH_PREFIX_CHECK_ON = - new CommonVars<>("wds.linkis.storage.hdfs.prefix_check.enable", true, null, null); - - public static CommonVars HDFS_PATH_PREFIX_REMOVE = - new CommonVars<>("wds.linkis.storage.hdfs.prefxi.remove", true, null, null); - - public static CommonVars FS_CACHE_DISABLE = - new CommonVars<>("wds.linkis.fs.hdfs.impl.disable.cache", false, null, null); - - public static CommonVars FS_CHECKSUM_DISBALE = - new CommonVars<>("linkis.fs.hdfs.impl.disable.checksum", false, null, null); - - /** - * more arguments please refer to: - * https://hadoop.apache.org/docs/stable/hadoop-aliyun/tools/hadoop-aliyun/index.html Aliyun OSS - * endpoint to connect to. eg: https://oss-cn-hangzhou.aliyuncs.com - */ - public static CommonVars OSS_ENDPOINT = - new CommonVars("wds.linkis.fs.oss.endpoint", "", null, null); - - /** Aliyun bucket name eg: benchmark2 */ - public static CommonVars OSS_ACCESS_BUCKET_NAME = - new CommonVars("wds.linkis.fs.oss.bucket.name", "", null, null); - - /** Aliyun access key ID */ - public static CommonVars OSS_ACCESS_KEY_ID = - new CommonVars("wds.linkis.fs.oss.accessKeyId", "", null, null); - - /** Aliyun access key secret */ - public static CommonVars OSS_ACCESS_KEY_SECRET = - new CommonVars("wds.linkis.fs.oss.accessKeySecret", "", null, null); - - public static CommonVars OSS_PATH_PREFIX_CHECK_ON = - new CommonVars("wds.linkis.storage.oss.prefix_check.enable", false, null, null); - - public static CommonVars OSS_PATH_PREFIX_REMOVE = - new CommonVars("wds.linkis.storage.oss.prefix.remove", true, null, null); - - public static CommonVars S3_ACCESS_KEY = - new CommonVars("linkis.storage.s3.access.key", "", null, null); - - public static CommonVars S3_SECRET_KEY = - new CommonVars("linkis.storage.s3.secret.key", "", null, null); - - public static CommonVars S3_ENDPOINT = - new CommonVars("linkis.storage.s3.endpoint", "", null, null); - - public static CommonVars S3_REGION = - new CommonVars("linkis.storage.s3.region", "", null, null); - - public static CommonVars S3_BUCKET = - new CommonVars("linkis.storage.s3.bucket", "", null, null); -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java deleted file mode 100644 index 491c3d7af4..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.utils; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.storage.FSFactory; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.ResultSetReaderFactory; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.io.IOException; -import java.util.Arrays; - -/** - * 工具类,用于做storage jar包打出来做测试用 Tool class, which is used to print the storage jar package for testing - */ -public class StorageHelper { - private static final Log logger = LogFactory.getLog(StorageHelper.class); - - public static void main(String[] args) { - if (args.length < 2) logger.info("Usage method params eg:getTableResLines path"); - String method = args[0]; - String[] params = Arrays.copyOfRange(args, 1, args.length); - try { - Thread.sleep(10000L); - } catch (InterruptedException e) { - } - - switch (method) { - case "getTableResLines": - getTableResLines(params); - break; - case "getTableRes": - getTableRes(params); - break; - case "createNewFile": - createNewFile(params); - break; - default: - logger.info("There is no such method"); - } - } - - /** - * Get the number of table result set file lines(获得表格结果集文件行数) - * - * @param args - */ - public static void getTableResLines(String[] args) { - ResultSetReader resultSetReader = null; - try { - FsPath resPath = StorageUtils.getFsPath(args[0]); - ResultSetFactory resultSetFactory = ResultSetFactory.getInstance(); - - ResultSet resultSet = - resultSetFactory.getResultSetByType(ResultSetFactory.TABLE_TYPE); - Fs fs = FSFactory.getFs(resPath); - fs.init(null); - resultSetReader = - ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath), resPath); - TableMetaData metaData = (TableMetaData) resultSetReader.getMetaData(); - Arrays.stream(metaData.getColumns()).forEach(column -> logger.info(column.toString())); - int num = 0; - Thread.sleep(10000L); - while (resultSetReader.hasNext()) { - resultSetReader.getRecord(); - num++; - } - logger.info(Integer.toString(num)); - } catch (Exception e) { - logger.error("getTableResLines error:", e); - } finally { - if (resultSetReader != null) { - try { - resultSetReader.close(); - } catch (IOException e) { - logger.error("Failed to close ResultSetReader", e); - } - } - } - } - - public static void getTableRes(String[] args) { - try { - int len = Integer.parseInt(args[1]); - int max = len + 10; - FsPath resPath = StorageUtils.getFsPath(args[0]); - ResultSetFactory resultSetFactory = ResultSetFactory.getInstance(); - ResultSet resultSet = - resultSetFactory.getResultSetByType(ResultSetFactory.TABLE_TYPE); - Fs fs = FSFactory.getFs(resPath); - - fs.init(null); - - ResultSetReader reader = - ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath), resPath); - MetaData rmetaData = reader.getMetaData(); - Arrays.stream(((TableMetaData) rmetaData).getColumns()) - .forEach(column -> logger.info(column.toString())); - Arrays.stream(((TableMetaData) rmetaData).getColumns()) - .map(column -> column.getColumnName() + ",") - .forEach(column -> logger.info(column)); - int num = 0; - while (reader.hasNext()) { - num++; - if (num > max) return; - if (num > len) { - Record record = reader.getRecord(); - Arrays.stream(((TableRecord) record).row) - .forEach( - value -> { - logger.info(value.toString()); - logger.info(","); - }); - logger.info("\n"); - } - } - } catch (IOException e) { - logger.warn("StorageHelper getTableRes failed", e); - } - } - - public static void createNewFile(String[] args) { - FsPath resPath = StorageUtils.getFsPath(args[0]); - String proxyUser = StorageUtils.getJvmUser(); - try { - FileSystemUtils.createNewFile(resPath, proxyUser, true); - } catch (Exception e) { - logger.warn("StorageHelper createNewFile failed", e); - } - logger.info("success"); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java deleted file mode 100644 index 07bc0510bc..0000000000 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.utils; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.common.io.resultset.ResultSetReader; -import org.apache.linkis.common.io.resultset.ResultSetWriter; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.hadoop.common.conf.HadoopConf; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; -import org.apache.linkis.storage.exception.StorageWarnException; -import org.apache.linkis.storage.resultset.ResultSetFactory; -import org.apache.linkis.storage.resultset.ResultSetReaderFactory; -import org.apache.linkis.storage.resultset.ResultSetWriterFactory; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.utils.CloseableUtils; - -import java.io.*; -import java.lang.reflect.Method; -import java.text.NumberFormat; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.CONFIGURATION_NOT_READ; - -public class StorageUtils { - private static final Logger logger = LoggerFactory.getLogger(StorageUtils.class); - - public static final String HDFS = "hdfs"; - public static final String FILE = "file"; - public static final String OSS = "oss"; - public static final String S3 = "s3"; - - public static final String FILE_SCHEMA = "file://"; - public static final String HDFS_SCHEMA = "hdfs://"; - public static final String OSS_SCHEMA = "oss://"; - public static final String S3_SCHEMA = "s3://"; - - private static final NumberFormat nf = NumberFormat.getInstance(); - - static { - nf.setGroupingUsed(false); - nf.setMaximumFractionDigits((int) StorageConfiguration.DOUBLE_FRACTION_LEN.getValue()); - } - - public static String doubleToString(double value) { - return nf.format(value); - } - - public static Map loadClass(String classStr, Function op) { - String[] _classes = classStr.split(","); - LinkedHashMap classes = new LinkedHashMap<>(); - for (String clazz : _classes) { - try { - T obj = Utils.getClassInstance(clazz.trim()); - classes.put(op.apply(obj), obj); - } catch (Exception e) { - logger.warn("StorageUtils loadClass failed", e); - } - } - return classes; - } - - public static Map> loadClasses( - String classStr, String pge, Function, String> op) { - String[] _classes = - StringUtils.isEmpty(pge) - ? classStr.split(",") - : Stream.of(StringUtils.split(classStr, ',')) - .map(value -> pge + "." + value) - .toArray(String[]::new); - Map> classes = new LinkedHashMap<>(); - for (String clazz : _classes) { - try { - Class _class = - (Class) Thread.currentThread().getContextClassLoader().loadClass(clazz.trim()); - classes.put(op.apply(_class), _class); - } catch (Exception e) { - logger.warn("StorageUtils loadClasses failed", e); - } - } - return classes; - } - - public static String pathToSuffix(String path) { - String fileName = new File(path).getName(); - if (fileName.length() > 0) { - int dot = fileName.lastIndexOf('.'); - if (dot > -1 && dot < fileName.length() - 1) { - return fileName.substring(dot + 1); - } - } - return fileName; - } - - public static Object invoke(Object obj, Method method, Object[] args) throws Exception { - return method.invoke(obj, args); - } - - /** - * Serialized string is a result set of type Text(序列化字符串为Text类型的结果集) - * - * @param value - * @return - */ - public static String serializerStringToResult(String value) throws IOException { - ResultSet resultSet = - ResultSetFactory.getInstance().getResultSetByType(ResultSetFactory.TEXT_TYPE); - ResultSetWriter writer = - ResultSetWriterFactory.getResultSetWriter(resultSet, Long.MAX_VALUE, null); - LineMetaData metaData = new LineMetaData(null); - LineRecord record = new LineRecord(value); - writer.addMetaData(metaData); - writer.addRecord(record); - String res = writer.toString(); - IOUtils.closeQuietly(writer); - return res; - } - - /** - * The result set of serialized text is a string(序列化text的结果集为字符串) - * - * @param result - * @return - */ - public static String deserializerResultToString(String result) throws IOException { - ResultSet resultSet = - ResultSetFactory.getInstance().getResultSetByType(ResultSetFactory.TEXT_TYPE); - ResultSetReader reader = ResultSetReaderFactory.getResultSetReader(resultSet, result); - StringBuilder sb = new StringBuilder(); - while (reader.hasNext()) { - LineRecord record = (LineRecord) reader.getRecord(); - sb.append(record.getLine()); - } - reader.close(); - return sb.toString(); - } - - public static void close(OutputStream outputStream) { - close(outputStream, null, null); - } - - public static void close(InputStream inputStream) { - close(null, inputStream, null); - } - - public static void close(Fs fs) { - close(null, null, fs); - } - - public static void close(OutputStream outputStream, InputStream inputStream, Fs fs) { - try { - if (outputStream != null) outputStream.close(); - } catch (IOException e) { - // ignore exception - } - try { - if (inputStream != null) inputStream.close(); - } catch (IOException e) { - // ignore exception - } - try { - if (fs != null) fs.close(); - } catch (IOException e) { - // ignore exception - } - } - - public static void close(Closeable closeable) { - CloseableUtils.closeQuietly(closeable); - } - - public static String getJvmUser() { - return System.getProperty("user.name"); - } - - public static boolean isHDFSNode() { - File confPath = new File(HadoopConf.hadoopConfDir()); - // TODO IO-client mode need return false - if (!confPath.exists() || confPath.isFile()) { - throw new StorageWarnException( - CONFIGURATION_NOT_READ.getErrorCode(), CONFIGURATION_NOT_READ.getErrorDesc()); - } else return true; - } - - /** - * Returns the FsPath by determining whether the path is a schema. By default, the FsPath of the - * file is returned. - * - * @param path - * @return - */ - public static FsPath getFsPath(String path) { - if (path.startsWith(FILE_SCHEMA) || path.startsWith(HDFS_SCHEMA)) { - return new FsPath(path); - } else { - return new FsPath(FILE_SCHEMA + path); - } - } - - public static int readBytes(InputStream inputStream, byte[] bytes, int len) { - int readLen = 0; - try { - int count = 0; - // 当使用s3存储结果文件时时，com.amazonaws.services.s3.model.S3InputStream无法正确读取.dolphin文件。需要在循环条件添加: - // readLen >= 0 - // To resolve the issue when using S3 to store result files and - // com.amazonaws.services.s3.model.S3InputStream to read .dolphin files, you need to add the - // condition readLen >= 0 in the loop. - while (readLen < len && readLen >= 0) { - count = inputStream.read(bytes, readLen, len - readLen); - - if (count == -1 && inputStream.available() < 1) { - return readLen; - } - readLen += count; - } - } catch (IOException e) { - logger.warn("FileSystemUtils readBytes failed", e); - } - return readLen; - } - - public static String colToString(Object col, String nullValue) { - if (col == null) { - return nullValue; - } else if (col instanceof Double) { - return doubleToString((Double) col); - } else if ("NULL".equals(col) || "".equals(col)) { - return nullValue; - } else { - return col.toString(); - } - } - - public static String colToString(Object col) { - return colToString(col, "NULL"); - } - - public static boolean isIOProxy() { - return (boolean) StorageConfiguration.ENABLE_IO_PROXY.getValue(); - } - - public static byte[] mergeByteArrays(byte[] arr1, byte[] arr2) { - byte[] mergedArray = new byte[arr1.length + arr2.length]; - System.arraycopy(arr1, 0, mergedArray, 0, arr1.length); - System.arraycopy(arr2, 0, mergedArray, arr1.length, arr2.length); - return mergedArray; - } - - public static boolean isHDFSPath(FsPath fsPath) { - return HDFS.equalsIgnoreCase(fsPath.getFsType()); - } -} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala new file mode 100644 index 0000000000..5968332e10 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/FSFactory.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage + +import org.apache.linkis.common.io.{Fs, FsPath} +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_FILE +import org.apache.linkis.storage.exception.StorageFatalException +import org.apache.linkis.storage.factory.BuildFactory +import org.apache.linkis.storage.utils.{StorageConfiguration, StorageUtils} + +import java.text.MessageFormat + +object FSFactory extends Logging { + + private val buildClasses: Map[String, BuildFactory] = StorageUtils.loadClass[BuildFactory]( + StorageConfiguration.STORAGE_BUILD_FS_CLASSES.getValue, + t => t.fsName() + ) + + def getBuildFactory(fsName: String): BuildFactory = { + if (!buildClasses.contains(fsName)) { + throw new StorageFatalException( + UNSUPPORTED_FILE.getErrorCode, + MessageFormat.format(UNSUPPORTED_FILE.getErrorDesc, fsName) + ) + } + buildClasses(fsName) + } + + def getFs(fsType: String, proxyUser: String): Fs = { + val user = StorageUtils.getJvmUser + getBuildFactory(fsType).getFs(user, proxyUser) + } + + def getFs(fsType: String): Fs = { + val user = StorageUtils.getJvmUser + getBuildFactory(fsType).getFs(user, user) + } + + /** + * 1. If this machine has shared storage, the file:// type FS obtained here is the FS of the + * process user. 2, if this machine does not have shared storage, then the file:// type FS + * obtained is the proxy to the Remote (shared storage machine root) FS 3. If it is HDFS, it + * returns the FS of the process user. 1、如果这台机器装有共享存储则这里获得的file://类型的FS为该进程用户的FS + * 2、如果这台机器没有共享存储则获得的file://类型的FS为代理到Remote（共享存储机器root）的FS 3、如果是HDFS则返回的就是该进程用户的FS + * @param fsPath + * @return + */ + def getFs(fsPath: FsPath): Fs = { + getFs(fsPath.getFsType()) + } + + /** + * 1. If the process user is passed and the proxy user and the process user are consistent, the + * file:// type FS is the FS of the process user (the shared storage exists) 2, if the + * process user is passed and the proxy user and the process user are consistent and there is + * no shared storage, the file:// type FS is the proxy to the remote (shared storage machine + * root) FS 3. If the passed proxy user and process user are consistent, the hdfs type is the + * FS of the process user. 4. If the proxy user and the process user are inconsistent, the + * hdfs type is the FS after the proxy. 1、如果传了进程用户且代理用户和进程用户一致则file://类型的FS为该进程用户的FS（存在共享存储） + * 2、如果传了进程用户且代理用户和进程用户一致且没有共享存储则file://类型的FS为代理到Remote（共享存储机器root）的FS + * 3、如果传了的代理用户和进程用户一致则hdfs类型为该进程用户的FS 4、如果传了代理用户和进程用户不一致则hdfs类型为代理后的FS + * + * @param fsPath + * @param proxyUser + * @return + */ + def getFsByProxyUser(fsPath: FsPath, proxyUser: String): Fs = { + getFs(fsPath.getFsType(), proxyUser) + } + + def getFSByLabel(fs: String, label: String): Fs = { + val user = StorageUtils.getJvmUser + getBuildFactory(fs).getFs(user, user, label) + } + + def getFSByLabelAndUser(fs: String, label: String, proxy: String): Fs = { + val user = StorageUtils.getJvmUser + getBuildFactory(fs).getFs(user, proxy, label) + } + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientFactoryTest.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineMetaData.scala similarity index 69% rename from linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientFactoryTest.scala rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineMetaData.scala index c284a14c33..e6c9df4548 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientFactoryTest.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineMetaData.scala @@ -15,22 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.filesystem +package org.apache.linkis.storage -import org.junit.jupiter.api.{Assertions, DisplayName, Test} +import org.apache.linkis.common.io.MetaData +import org.apache.linkis.storage.resultset.ResultMetaData -class WorkspaceClientFactoryTest { +class LineMetaData(private var metaData: String = null) extends ResultMetaData { - @Test - @DisplayName("getClientTest") - def getClientTest(): Unit = { + def getMetaData: String = metaData - val user = "hadoop" - val token = "abc" - val gatewayAddress = "127.0.0.1:9001" - - val client = WorkspaceClientFactory.getClient(user, token, gatewayAddress) - Assertions.assertNotNull(client) + def setMetaData(metaData: String): Unit = { + this.metaData = metaData } + override def cloneMeta(): MetaData = new LineMetaData(metaData) } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/OperatorFactory.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineRecord.scala similarity index 67% rename from linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/OperatorFactory.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineRecord.scala index e4d02f945b..d6e3220cf2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/OperatorFactory.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/LineRecord.scala @@ -15,19 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.manager.common.operator; +package org.apache.linkis.storage -import java.util.Map; +import org.apache.linkis.common.io.Record +import org.apache.linkis.storage.resultset.ResultRecord -public interface OperatorFactory { +class LineRecord(private var line: String) extends ResultRecord { - String getOperatorName(Map parameters); + def getLine: String = line - Operator getOperatorRequest(Map parameters); + def setLine(line: String): Unit = { + this.line = line + } - OperatorFactory operatorFactory = new OperatorFactoryImpl(); + override def cloneRecord(): Record = new LineRecord(line) - static OperatorFactory apply() { - return operatorFactory; - } + override def toString: String = line } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala new file mode 100644 index 0000000000..5330983dd6 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.conf + +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.ByteTimeUtils + +import org.apache.commons.lang3.StringUtils + +object LinkisStorageConf { + private val CONF_LOCK = new Object() + + val enableLimitThreadLocal: ThreadLocal[String] = new ThreadLocal[String] + + val columnIndicesThreadLocal: ThreadLocal[Array[Int]] = new ThreadLocal[Array[Int]] + + val HDFS_FILE_SYSTEM_REST_ERRS: String = + CommonVars + .apply( + "wds.linkis.hdfs.rest.errs", + ".*Filesystem closed.*|.*Failed to find any Kerberos tgt.*" + ) + .getValue + + val ROW_BYTE_MAX_LEN_STR = CommonVars("wds.linkis.resultset.row.max.str", "2m").getValue + + val ROW_BYTE_MAX_LEN = ByteTimeUtils.byteStringAsBytes(ROW_BYTE_MAX_LEN_STR) + + val FILE_TYPE = CommonVars( + "wds.linkis.storage.file.type", + "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql,tsql,txt" + ).getValue + + private var fileTypeArr: Array[String] = null + + private def fileTypeArrParser(fileType: String): Array[String] = { + if (StringUtils.isBlank(fileType)) Array() + else fileType.split(",") + } + + def getFileTypeArr: Array[String] = { + if (fileTypeArr == null) { + CONF_LOCK.synchronized { + if (fileTypeArr == null) { + fileTypeArr = fileTypeArrParser(FILE_TYPE) + } + } + } + fileTypeArr + } + + val LINKIS_RESULT_ENABLE_NULL = CommonVars("linkis.resultset.enable.null.replace", false).getValue + + val LINKIS_RESULT_COLUMN_SIZE = + CommonVars("linkis.resultset.column.size.max", Int.MaxValue).getValue + + val LINKIS_RESULT_COL_LENGTH = + CommonVars("linkis.resultset.col.length.max", Int.MaxValue).getValue + + val LINKIS__READ_RESULT_ROW_MAX_LEN_STR = + CommonVars("linkis.resultset.read.row.max.str", "20m").getValue + + val LINKIS_READ_ROW_BYTE_MAX_LEN = + ByteTimeUtils.byteStringAsBytes(LINKIS__READ_RESULT_ROW_MAX_LEN_STR) + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/entity/ResourceOperationType.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsReader.scala similarity index 85% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/entity/ResourceOperationType.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsReader.scala index c1a4f211b4..d40d041a3e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/entity/ResourceOperationType.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsReader.scala @@ -15,11 +15,8 @@ * limitations under the License. */ -package org.apache.linkis.manager.rm.entity; +package org.apache.linkis.storage.csv -public enum ResourceOperationType { - LOCK, - USED, - USED_RELEASE, - LOCKER_RELEASE -} +import org.apache.linkis.common.io.FsReader + +abstract class CSVFsReader extends FsReader {} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala new file mode 100644 index 0000000000..93610a7db2 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/CSVFsWriter.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.csv + +import org.apache.linkis.common.io.FsWriter + +import java.io.OutputStream + +abstract class CSVFsWriter extends FsWriter { + val charset: String + val separator: String + val quoteRetouchEnable: Boolean +} + +object CSVFsWriter { + + def getCSVFSWriter( + charset: String, + separator: String, + quoteRetouchEnable: Boolean, + outputStream: OutputStream + ): CSVFsWriter = new StorageCSVWriter(charset, separator, quoteRetouchEnable, outputStream) + + def getCSVFSWriter( + charset: String, + separator: String, + quoteRetouchEnable: Boolean, + outputStream: OutputStream, + keepNewline: Boolean + ): CSVFsWriter = + new StorageCSVWriter(charset, separator, quoteRetouchEnable, outputStream, keepNewline) + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/AlertUtils.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVReader.scala similarity index 91% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/AlertUtils.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVReader.scala index a06dfaab18..6fc91c836b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/AlertUtils.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVReader.scala @@ -15,6 +15,6 @@ * limitations under the License. */ -package org.apache.linkis.manager.rm.utils; +package org.apache.linkis.storage.csv -public class AlertUtils {} +class StorageCSVReader {} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala new file mode 100644 index 0000000000..95d98669b8 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.csv + +import org.apache.linkis.common.io.{MetaData, Record} +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.storage.domain.DataType +import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} + +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.StringUtils + +import java.io._ + +class StorageCSVWriter( + val charset: String, + val separator: String, + val quoteRetouchEnable: Boolean, + val outputStream: OutputStream +) extends CSVFsWriter + with Logging { + + var keepNewline: Boolean = false; + + def this( + charset: String, + separator: String, + quoteRetouchEnable: Boolean, + outputStream: OutputStream, + keepNewline: Boolean + ) { + this(charset, separator, quoteRetouchEnable, outputStream) + this.keepNewline = keepNewline + } + + private val delimiter = separator match { + // Compatible with possible missing escape characters + case "t" => '\t' + case separ if StringUtils.isNotEmpty(separ) => separ + case _ => '\t' + } + + private val buffer: StringBuilder = new StringBuilder(50000) + + @scala.throws[IOException] + override def addMetaData(metaData: MetaData): Unit = { + val head = metaData.asInstanceOf[TableMetaData].columns.map(_.columnName) + write(head) + } + + private def compact(row: Array[String]): String = { + val quotationMarks: String = "\"" + val dealNewlineSymbolMarks: String = "\n" + + def decorateValue(v: String): String = { + if (StringUtils.isBlank(v)) v + else { + var res = v + if (quoteRetouchEnable) { + res = s"$quotationMarks${v.replaceAll(quotationMarks, "")}$quotationMarks" + } + if (!this.keepNewline) { + res = res.replaceAll(dealNewlineSymbolMarks, " ") + } + logger.debug("decorateValue with input:" + v + " output:" + res) + res + } + } + + if (logger.isDebugEnabled()) { + logger.debug("delimiter:" + delimiter.toString) + } + + row.map(x => decorateValue(x)).toList.mkString(delimiter.toString) + "\n" + } + + private def write(row: Array[String]) = { + val content: String = compact(row) + if (buffer.length + content.length > 49500) { + IOUtils.write(buffer.toString().getBytes(charset), outputStream) + buffer.clear() + } + buffer.append(content) + } + + @scala.throws[IOException] + override def addRecord(record: Record): Unit = { + val body = record.asInstanceOf[TableRecord].row.map(DataType.valueToString) + write(body) + } + + override def flush(): Unit = { + IOUtils.write(buffer.toString().getBytes(charset), outputStream) + buffer.clear() + } + + override def close(): Unit = { + flush() + IOUtils.closeQuietly(outputStream) + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala new file mode 100644 index 0000000000..036cd8bfa2 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.domain + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.conf.LinkisStorageConf + +import java.math.{BigDecimal => JavaBigDecimal} +import java.sql.{Date, Timestamp} + +object DataType extends Logging { + + val LOWCASE_NULL_VALUE = "null" + + val DECIMAL_REGEX = "^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)".r.unanchored + + val SHORT_REGEX = "^short.*".r.unanchored + val INT_REGEX = "^int.*".r.unanchored + val LONG_REGEX = "^long.*".r.unanchored + val BIGINT_REGEX = "^bigint.*".r.unanchored + val FLOAT_REGEX = "^float.*".r.unanchored + val DOUBLE_REGEX = "^double.*".r.unanchored + + val VARCHAR_REGEX = "^varchar.*".r.unanchored + val CHAR_REGEX = "^char.*".r.unanchored + + val ARRAY_REGEX = "array.*".r.unanchored + + val MAP_REGEX = "map.*".r.unanchored + + val LIST_REGEX = "list.*".r.unanchored + + val STRUCT_REGEX = "struct.*".r.unanchored + + implicit def toDataType(dataType: String): DataType = dataType match { + case "void" | "null" => NullType + case "string" => StringType + case "boolean" => BooleanType + case SHORT_REGEX() => ShortIntType + case LONG_REGEX() => LongType + case BIGINT_REGEX() => BigIntType + case INT_REGEX() | "integer" | "smallint" => IntType + case FLOAT_REGEX() => FloatType + case DOUBLE_REGEX() => DoubleType + case VARCHAR_REGEX() => VarcharType + case CHAR_REGEX() => CharType + case "date" => DateType + case "timestamp" => TimestampType + case "binary" => BinaryType + case "decimal" | DECIMAL_REGEX() => DecimalType + case ARRAY_REGEX() => ArrayType + case MAP_REGEX() => MapType + case LIST_REGEX() => ListType + case STRUCT_REGEX() => StructType + case _ => StringType + } + + def toValue(dataType: DataType, value: String): Any = { + var newValue: String = value + if (isLinkisNull(value)) { + if (!LinkisStorageConf.LINKIS_RESULT_ENABLE_NULL) { + return null + } else { + newValue = Dolphin.NULL + } + } + Utils.tryCatch(dataType match { + case NullType => null + case StringType | CharType | VarcharType | StructType | ListType | ArrayType | MapType => + newValue + case BooleanType => if (isNumberNull(newValue)) null else newValue.toBoolean + case ShortIntType => if (isNumberNull(newValue)) null else newValue.toShort + case IntType => if (isNumberNull(newValue)) null else newValue.toInt + case LongType | BigIntType => if (isNumberNull(newValue)) null else newValue.toLong + case FloatType => if (isNumberNull(newValue)) null else newValue.toFloat + case DoubleType => if (isNumberNull(newValue)) null else newValue.toDouble + case DecimalType => if (isNumberNull(newValue)) null else new JavaBigDecimal(newValue) + case DateType => if (isNumberNull(newValue)) null else Date.valueOf(newValue) + case TimestampType => + if (isNumberNull(newValue)) null else Timestamp.valueOf(newValue).toString.stripSuffix(".0") + case BinaryType => if (isNull(newValue)) null else newValue.getBytes() + case _ => newValue + }) { t => + logger.debug(s"Failed to $newValue switch to dataType:", t) + newValue + } + } + + def isLinkisNull(value: String): Boolean = { + if (value == null || value == Dolphin.LINKIS_NULL) true else false + } + + def isNull(value: String): Boolean = + if (value == null || value == Dolphin.NULL || value.trim == "") true else false + + def isNumberNull(value: String): Boolean = + if (null == value || Dolphin.NULL.equalsIgnoreCase(value) || value.trim == "") { + true + } else { + false + } + + def valueToString(value: Any): String = { + if (null == value) return null + value match { + case javaDecimal: JavaBigDecimal => + javaDecimal.toPlainString + case _ => value.toString + } + } + +} + +abstract class DataType(val typeName: String, val javaSQLType: Int) { + override def toString: String = typeName +} + +case object NullType extends DataType("void", 0) +case object StringType extends DataType("string", 12) +case object BooleanType extends DataType("boolean", 16) +case object TinyIntType extends DataType("tinyint", -6) +case object ShortIntType extends DataType("short", 5) +case object IntType extends DataType("int", 4) +case object LongType extends DataType("long", -5) +case object BigIntType extends DataType("bigint", -5) +case object FloatType extends DataType("float", 6) +case object DoubleType extends DataType("double", 8) +case object CharType extends DataType("char", 1) +case object VarcharType extends DataType("varchar", 12) +case object DateType extends DataType("date", 91) +case object TimestampType extends DataType("timestamp", 93) +case object BinaryType extends DataType("binary", -2) +case object DecimalType extends DataType("decimal", 3) +case object ArrayType extends DataType("array", 2003) +case object MapType extends DataType("map", 2000) +case object ListType extends DataType("list", 2001) +case object StructType extends DataType("struct", 2002) +case object BigDecimalType extends DataType("bigdecimal", 3) + +case class Column(columnName: String, dataType: DataType, comment: String) { + + def toArray: Array[Any] = { + Array[Any](columnName, dataType, comment) + } + + override def toString: String = + s"columnName:$columnName,dataType:$dataType,comment:$comment" + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala new file mode 100644 index 0000000000..667f1a635e --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/Dolphin.scala @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.domain + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.FAILED_TO_READ_INTEGER +import org.apache.linkis.storage.exception.{ + StorageErrorCode, + StorageErrorException, + StorageWarnException +} +import org.apache.linkis.storage.utils.{StorageConfiguration, StorageUtils} + +import java.io.{InputStream, IOException} + +object Dolphin extends Logging { + + val CHAR_SET = StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue + val MAGIC = "dolphin" + + val MAGIC_BYTES = MAGIC.getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue) + val MAGIC_LEN = MAGIC_BYTES.length + + val DOLPHIN_FILE_SUFFIX = ".dolphin" + + val COL_SPLIT = "," + val COL_SPLIT_BYTES = COL_SPLIT.getBytes("utf-8") + val COL_SPLIT_LEN = COL_SPLIT_BYTES.length + + val NULL = "NULL" + val NULL_BYTES = NULL.getBytes("utf-8") + + val LINKIS_NULL = "LINKIS_NULL" + val LINKIS_NULL_BYTES = LINKIS_NULL.getBytes("utf-8") + + val INT_LEN = 10 + + val FILE_EMPTY = 31 + + def getBytes(value: Any): Array[Byte] = { + value.toString.getBytes(CHAR_SET) + } + + /** + * Convert a bytes array to a String content 将bytes数组转换为String内容 + * @param bytes + * @param start + * @param len + * @return + */ + def getString(bytes: Array[Byte], start: Int, len: Int): String = { + try { + new String(bytes, start, len, Dolphin.CHAR_SET) + } catch { + case e: OutOfMemoryError => + logger.error("bytes to String oom {} Byte", bytes.length) + throw new StorageErrorException( + StorageErrorCode.FS_OOM.getCode, + StorageErrorCode.FS_OOM.getMessage, + e + ) + } + } + + def toStringValue(value: String): String = { + if (LINKIS_NULL.equals(value)) { + NULL + } else { + value + } + } + + /** + * Read an integer value that converts the array to a byte of length 10 bytes + * 读取整数值，该值为将数组转换为10字节长度的byte + * @param inputStream + * @return + */ + def readInt(inputStream: InputStream): Int = { + val bytes = new Array[Byte](INT_LEN + 1) + if (StorageUtils.readBytes(inputStream, bytes, INT_LEN) != INT_LEN) { + throw new StorageWarnException( + FAILED_TO_READ_INTEGER.getErrorCode, + FAILED_TO_READ_INTEGER.getErrorDesc + ) + } + getString(bytes, 0, INT_LEN).toInt + } + + /** + * Print integers at a fixed length(将整数按固定长度打印) + * @param value + * @return + */ + def getIntBytes(value: Int): Array[Byte] = { + val str = value.toString + val res = "0" * (INT_LEN - str.length) + str + Dolphin.getBytes(res) + } + + def getType(inputStream: InputStream): String = { + val bytes = new Array[Byte](100) + val len = StorageUtils.readBytes(inputStream, bytes, Dolphin.MAGIC_LEN + INT_LEN) + if (len == -1) return null + getType(Dolphin.getString(bytes, 0, len)) + } + + def getType(content: String): String = { + if (content.length < MAGIC.length || content.substring(0, MAGIC.length) != MAGIC) { + throw new IOException(s"File header type must be dolphin,content:$content is not") + } + content.substring(MAGIC.length, MAGIC.length + INT_LEN).toInt.toString + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala new file mode 100644 index 0000000000..fac0a2d01b --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.domain + +import java.lang.reflect.Type + +import com.google.gson.GsonBuilder + +/** + * @param id + * Engine unique Id(engine唯一的Id) + * @param fsType + * Fs type(fs类型) + * @param creatorUser + * Create a user to start the corresponding jvm user(创建用户为对应启动的jvm用户) + * @param proxyUser + * Proxy user(代理用户) + * @param clientIp + * client Ip for whitelist control(ip用于白名单控制) + * @param methodName + * Method name called(调用的方法名) + * @param params + * Method parameter(方法参数) + */ +case class MethodEntity( + id: Long, + fsType: String, + creatorUser: String, + proxyUser: String, + clientIp: String, + methodName: String, + params: Array[AnyRef] +) { + + override def toString: String = { + s"id:$id, methodName:$methodName, fsType:$fsType, " + + s"creatorUser:$creatorUser, proxyUser:$proxyUser, clientIp:$clientIp, " + } + +} + +object MethodEntitySerializer { + + val gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create + + /** + * Serialized to code as a MethodEntity object 序列化为code为MethodEntity对象 + * @param code + * @return + */ + def deserializer(code: String): MethodEntity = gson.fromJson(code, classOf[MethodEntity]) + + /** + * Serialize MethodEntity to code 序列化MethodEntity为code + * @param methodEntity + * @return + */ + def serializer(methodEntity: MethodEntity): String = gson.toJson(methodEntity) + + /** + * Serialize a java object as a string 序列化java对象为字符串 + * @param value + * @return + */ + def serializerJavaObject(value: Any): String = gson.toJson(value) + + /** + * Deserialize a string into a java object 将字符串解序列化为java对象 + * @param json + * @param classType + * @tparam T + * @return + */ + def deserializerToJavaObject[T](json: String, classType: Class[T]): T = { + gson.fromJson(json, classType) + } + + def deserializerToJavaObject[T](json: String, oType: Type): T = { + gson.fromJson(json, oType) + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsReader.scala new file mode 100644 index 0000000000..621145cb4d --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsReader.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.excel + +import org.apache.linkis.common.io.FsReader + +abstract class ExcelFsReader extends FsReader {} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptMetaData.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala similarity index 59% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptMetaData.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala index 28999aae75..079920bef3 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptMetaData.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelFsWriter.scala @@ -15,27 +15,33 @@ * limitations under the License. */ -package org.apache.linkis.storage.script; +package org.apache.linkis.storage.excel -import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.FsWriter -public class ScriptMetaData implements MetaData { - private Variable[] variables; +import java.io.OutputStream - public ScriptMetaData(Variable[] variables) { - this.variables = variables; - } +abstract class ExcelFsWriter extends FsWriter { + val charset: String + val sheetName: String + val dateFormat: String + val autoFormat: Boolean +} - @Override - public MetaData cloneMeta() { - return new ScriptMetaData(variables.clone()); - } +object ExcelFsWriter { - public Variable[] getMetaData() { - return variables; - } + def getExcelFsWriter( + charset: String, + sheetName: String, + dateFormat: String, + outputStream: OutputStream, + autoFormat: Boolean + ): ExcelFsWriter = new StorageExcelWriter( + charset, + sheetName, + dateFormat, + outputStream: OutputStream, + autoFormat: Boolean + ) - public void setMetaData(Variable[] variables) { - this.variables = variables; - } } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelXlsReader.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelXlsReader.java new file mode 100644 index 0000000000..aac0ff54a6 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/ExcelXlsReader.java @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.excel; + +import org.apache.poi.hssf.eventusermodel.EventWorkbookBuilder.SheetRecordCollectingListener; +import org.apache.poi.hssf.eventusermodel.*; +import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord; +import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord; +import org.apache.poi.hssf.model.HSSFFormulaParser; +import org.apache.poi.hssf.record.*; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + + +public class ExcelXlsReader implements HSSFListener { + private int minColumns = -1; + + private POIFSFileSystem fs; + + private InputStream inputStream; + + private int lastRowNumber; + + private int lastColumnNumber; + + /** + * Should we output the formula, or the value it has? + */ + private boolean outputFormulaValues = true; + + /** + * For parsing Formulas + */ + private SheetRecordCollectingListener workbookBuildingListener; + + // excel2003Workbook(工作薄) + private HSSFWorkbook stubWorkbook; + + // Records we pick up as we process + private SSTRecord sstRecord; + + private FormatTrackingHSSFListener formatListener; + + // Table index(表索引) + private int sheetIndex = -1; + + private BoundSheetRecord[] orderedBSRs; + + @SuppressWarnings("unchecked") + private ArrayList boundSheetRecords = new ArrayList(); + + // For handling formulas with string results + private int nextRow; + + private int nextColumn; + + private boolean outputNextStringRecord; + + // Current line(当前行) + private int curRow = 0; + + // a container that stores row records(存储行记录的容器) + private List rowlist = new ArrayList(); + + + @SuppressWarnings("unused") + private String sheetName; + + private IExcelRowDeal excelRowDeal; + + + public void init(IExcelRowDeal excelRowDeal, InputStream inputStream) { + this.excelRowDeal = excelRowDeal; + this.inputStream = inputStream; + } + + + /** + * Traverse all the sheets under excel + * 遍历excel下所有的sheet + * @throws IOException + */ + public void process() throws IOException { + this.fs = new POIFSFileSystem(this.inputStream); + MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this); + formatListener = new FormatTrackingHSSFListener(listener); + HSSFEventFactory factory = new HSSFEventFactory(); + HSSFRequest request = new HSSFRequest(); + if (outputFormulaValues) { + request.addListenerForAllRecords(formatListener); + } else { + workbookBuildingListener = new SheetRecordCollectingListener(formatListener); + request.addListenerForAllRecords(workbookBuildingListener); + } + factory.processWorkbookEvents(request, fs); + } + + /** + * HSSFListener listener method, processing Record + * HSSFListener 监听方法，处理 Record + */ + @Override + @SuppressWarnings("unchecked") + public void processRecord(Record record) { + int thisRow = -1; + int thisColumn = -1; + String thisStr = null; + String value = null; + switch (record.getSid()) { + case BoundSheetRecord.sid: + boundSheetRecords.add(record); + break; + case BOFRecord.sid: + BOFRecord br = (BOFRecord) record; + if (br.getType() == BOFRecord.TYPE_WORKSHEET) { + // Create a child workbook if needed(如果有需要，则建立子工作薄) + if (workbookBuildingListener != null && stubWorkbook == null) { + stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook(); + } + + sheetIndex++; + if (orderedBSRs == null) { + orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords); + } + sheetName = orderedBSRs[sheetIndex].getSheetname(); + } + break; + + case SSTRecord.sid: + sstRecord = (SSTRecord) record; + break; + + case BlankRecord.sid: + BlankRecord brec = (BlankRecord) record; + thisRow = brec.getRow(); + thisColumn = brec.getColumn(); + thisStr = ""; + rowlist.add(thisColumn, thisStr); + break; + case BoolErrRecord.sid: // Cell is boolean(单元格为布尔类型) + BoolErrRecord berec = (BoolErrRecord) record; + thisRow = berec.getRow(); + thisColumn = berec.getColumn(); + thisStr = berec.getBooleanValue() + ""; + rowlist.add(thisColumn, thisStr); + break; + + case FormulaRecord.sid: // Cell is a formula type(单元格为公式类型) + FormulaRecord frec = (FormulaRecord) record; + thisRow = frec.getRow(); + thisColumn = frec.getColumn(); + if (outputFormulaValues) { + if (Double.isNaN(frec.getValue())) { + // Formula result is a string + // This is stored in the next record + outputNextStringRecord = true; + nextRow = frec.getRow(); + nextColumn = frec.getColumn(); + } else { + thisStr = formatListener.formatNumberDateCell(frec); + } + } else { + thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"'; + } + rowlist.add(thisColumn, thisStr); + break; + case StringRecord.sid:// a string of formulas in a cell(单元格中公式的字符串) + if (outputNextStringRecord) { + // String for formula + StringRecord srec = (StringRecord) record; + thisStr = srec.getString(); + thisRow = nextRow; + thisColumn = nextColumn; + outputNextStringRecord = false; + } + break; + case LabelRecord.sid: + LabelRecord lrec = (LabelRecord) record; + curRow = thisRow = lrec.getRow(); + thisColumn = lrec.getColumn(); + value = lrec.getValue().trim(); + value = value.equals("") ? " " : value; + this.rowlist.add(thisColumn, value); + break; + case LabelSSTRecord.sid: // Cell is a string type(单元格为字符串类型) + LabelSSTRecord lsrec = (LabelSSTRecord) record; + curRow = thisRow = lsrec.getRow(); + thisColumn = lsrec.getColumn(); + if (sstRecord == null) { + rowlist.add(thisColumn, " "); + } else { + value = sstRecord.getString(lsrec.getSSTIndex()).toString().trim(); + value = value.equals("") ? " " : value; + rowlist.add(thisColumn, value); + } + break; + case NumberRecord.sid: // Cell is a numeric type(单元格为数字类型) + NumberRecord numrec = (NumberRecord) record; + curRow = thisRow = numrec.getRow(); + thisColumn = numrec.getColumn(); + value = formatListener.formatNumberDateCell(numrec).trim(); + value = value.equals("") ? "0" : value; + // Add column values to the container(向容器加入列值) + rowlist.add(thisColumn, value); + break; + default: + break; + } + + // Encountered a new line of operations(遇到新行的操作)( + if (thisRow != -1 && thisRow != lastRowNumber) { + lastColumnNumber = -1; + } + + // Null operation(空值的操作) + if (record instanceof MissingCellDummyRecord) { + MissingCellDummyRecord mc = (MissingCellDummyRecord) record; + curRow = thisRow = mc.getRow(); + thisColumn = mc.getColumn(); + rowlist.add(thisColumn, " "); + } + + // Update row and column values(更新行和列的值) + if (thisRow > -1) + lastRowNumber = thisRow; + if (thisColumn > -1) + lastColumnNumber = thisColumn; + + // End of line operation(行结束时的操作) + if (record instanceof LastCellOfRowDummyRecord) { + if (minColumns > 0) { + // Column value is re-empted(列值重新置空) + if (lastColumnNumber == -1) { + lastColumnNumber = 0; + } + } + lastColumnNumber = -1; + + // At the end of each line, the dealRow() method(每行结束时， dealRow() 方法) + excelRowDeal.dealRow(orderedBSRs, sheetIndex, curRow, rowlist); + // Empty container(清空容器) + rowlist.clear(); + } + } + + public void close() { + try { + if (fs != null) { + fs.close(); + } + if (inputStream != null) { + inputStream.close(); + } + } catch (Exception e) { + + } + } + + public static void main(String[] args) { + + } +} + +class ExcelAnalysisException extends RuntimeException { + public ExcelAnalysisException() { + } + + public ExcelAnalysisException(String message) { + super(message); + } + + public ExcelAnalysisException(String message, Throwable cause) { + super(message, cause); + } + + public ExcelAnalysisException(Throwable cause) { + super(cause); + } +} + +interface IExcelRowDeal { + void dealRow(BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List rowlist); +} + + +class FirstRowDeal implements IExcelRowDeal { + + private List sheetNames = new ArrayList<>(); + private List row; + + public List getSheetNames() { + return sheetNames; + } + + public void setSheetNames(List sheetNames) { + this.sheetNames = sheetNames; + } + + public List getRow() { + return row; + } + + public void setRow(List row) { + this.row = row; + } + + @Override + public void dealRow(BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List rowlist) { + for (BoundSheetRecord record : orderedBSRs) { + sheetNames.add(record.getSheetname()); + } + row = rowlist; + throw new ExcelAnalysisException("Finished to deal first row"); + } + +} + +class RowToCsvDeal implements IExcelRowDeal { + + private Map params; + private List sheetNames; + private OutputStream outputStream; + private Boolean hasHeader; + private Boolean fisrtRow = true; + public void init(Boolean hasHeader, List sheetNames, OutputStream outputStream) { + this.hasHeader = hasHeader; + this.sheetNames = sheetNames; + this.outputStream = outputStream; + } + + @Override + public void dealRow(BoundSheetRecord[] orderedBSRs, int sheetIndex, int curRow, List rowlist) { + String sheetName = orderedBSRs[sheetIndex].getSheetname(); + if (sheetNames == null || sheetNames.isEmpty() || sheetNames.contains(sheetName)) { + if (! (curRow == 0 && hasHeader)) { + try { + if (fisrtRow) { + fisrtRow = false; + } else { + outputStream.write("\n".getBytes()); + } + int len = rowlist.size(); + for (int i = 0; i < len; i ++) { + outputStream.write(rowlist.get(i).replaceAll("\n|\t", " ").getBytes("utf-8")); + if (i < len -1) { + outputStream.write("\t".getBytes()); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResultResource.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelReader.scala similarity index 91% rename from linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResultResource.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelReader.scala index 0361b43af3..4a4f9bdf16 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResultResource.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelReader.scala @@ -15,6 +15,6 @@ * limitations under the License. */ -package org.apache.linkis.manager.rm; +package org.apache.linkis.storage.excel -public interface ResultResource {} +class StorageExcelReader {} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala new file mode 100644 index 0000000000..9ea83130df --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.excel + +import org.apache.linkis.common.io.{MetaData, Record} +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.domain._ +import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} + +import org.apache.commons.io.IOUtils +import org.apache.poi.ss.usermodel._ +import org.apache.poi.xssf.streaming.{SXSSFCell, SXSSFSheet, SXSSFWorkbook} + +import java.io._ +import java.math.BigDecimal +import java.util +import java.util.Date + +import scala.collection.mutable.ArrayBuffer + +class StorageExcelWriter( + val charset: String, + val sheetName: String, + val dateFormat: String, + val outputStream: OutputStream, + val autoFormat: Boolean +) extends ExcelFsWriter + with Logging { + + protected var workBook: SXSSFWorkbook = _ + protected var sheet: SXSSFSheet = _ + protected var format: DataFormat = _ + protected var types: Array[DataType] = _ + protected var rowPoint = 0 + protected var columnCounter = 0 + protected val styles = new util.HashMap[String, CellStyle]() + protected var isFlush = true + protected val os = new ByteArrayOutputStream() + protected var is: ByteArrayInputStream = _ + + def init: Unit = { + workBook = new SXSSFWorkbook() + sheet = workBook.createSheet(sheetName) + } + + def getDefaultHeadStyle: CellStyle = { + val headerFont = workBook.createFont + headerFont.setBold(true) + headerFont.setFontHeightInPoints(14.toShort) + headerFont.setColor(IndexedColors.BLACK.getIndex) + val headerCellStyle = workBook.createCellStyle + headerCellStyle.setFont(headerFont) + headerCellStyle + } + + def getWorkBook: Workbook = { + // 自适应列宽 + sheet.trackAllColumnsForAutoSizing() + for (elem <- 0 to columnCounter) { + sheet.autoSizeColumn(elem) + } + workBook + } + + def createCellStyle(dataType: DataType): CellStyle = { + val style = workBook.createCellStyle() + format = workBook.createDataFormat() + dataType.toString match { + case _ => style.setDataFormat(format.getFormat("@")) + } + if (autoFormat) { + dataType match { + case StringType => style.setDataFormat(format.getFormat("@")) + case TinyIntType => style.setDataFormat(format.getFormat("#")) + case ShortIntType => style.setDataFormat(format.getFormat("#")) + case IntType => style.setDataFormat(format.getFormat("#")) + case LongType => style.setDataFormat(format.getFormat("#.##E+00")) + case BigIntType => style.setDataFormat(format.getFormat("#.##E+00")) + case FloatType => style.setDataFormat(format.getFormat("#.0000000000")) + case DoubleType => style.setDataFormat(format.getFormat("#.0000000000")) + case CharType => style.setDataFormat(format.getFormat("@")) + case VarcharType => style.setDataFormat(format.getFormat("@")) + case DateType => style.setDataFormat(format.getFormat("m/d/yy h:mm")) + case TimestampType => style.setDataFormat(format.getFormat("m/d/yy h:mm")) + case DecimalType => style.setDataFormat(format.getFormat("#.000000000")) + case BigDecimalType => style.setDataFormat(format.getFormat("#.000000000")) + case _ => style.setDataFormat(format.getFormat("@")) + } + } + style + } + + def getCellStyle(dataType: DataType): CellStyle = { + val style = styles.get(dataType.typeName) + if (style == null) { + val newStyle = createCellStyle(dataType) + styles.put(dataType.typeName, newStyle) + newStyle + } else { + style + } + } + + @scala.throws[IOException] + override def addMetaData(metaData: MetaData): Unit = { + init + val tableHead = sheet.createRow(0) + val columns = metaData.asInstanceOf[TableMetaData].columns + val columnType = new ArrayBuffer[DataType]() + for (elem <- columns) { + val headCell = tableHead.createCell(columnCounter) + headCell.setCellValue(elem.columnName) + headCell.setCellStyle(getDefaultHeadStyle) + columnType += elem.dataType + columnCounter += 1 + } + types = columnType.toArray + rowPoint += 1 + } + + @scala.throws[IOException] + override def addRecord(record: Record): Unit = { + // TODO: 是否需要替换null值 + val tableBody = sheet.createRow(rowPoint) + var colunmPoint = 0 + val excelRecord = record.asInstanceOf[TableRecord].row + for (elem <- excelRecord) { + val cell = tableBody.createCell(colunmPoint) + val dataType = types.apply(colunmPoint) + if (autoFormat) { + setCellTypeValue(dataType, elem, cell) + } else { + cell.setCellValue(DataType.valueToString(elem)) + } + cell.setCellStyle(getCellStyle(dataType)) + colunmPoint += 1 + } + rowPoint += 1 + } + + private def setCellTypeValue(dataType: DataType, elem: Any, cell: SXSSFCell): Unit = { + if (null == elem) return + Utils.tryCatch { + dataType match { + case StringType => cell.setCellValue(DataType.valueToString(elem)) + case TinyIntType => cell.setCellValue(elem.toString.toInt) + case ShortIntType => cell.setCellValue(elem.toString.toInt) + case IntType => cell.setCellValue(elem.toString.toInt) + case LongType => cell.setCellValue(elem.toString.toLong) + case BigIntType => cell.setCellValue(elem.toString.toLong) + case FloatType => cell.setCellValue(elem.toString.toFloat) + case DoubleType => + doubleCheck(elem.toString) + cell.setCellValue(elem.toString.toDouble) + case CharType => cell.setCellValue(DataType.valueToString(elem)) + case VarcharType => cell.setCellValue(DataType.valueToString(elem)) + case DateType => cell.setCellValue(getDate(elem)) + case TimestampType => cell.setCellValue(getDate(elem)) + case DecimalType => + doubleCheck(DataType.valueToString(elem)) + cell.setCellValue(DataType.valueToString(elem).toDouble) + case BigDecimalType => + doubleCheck(DataType.valueToString(elem)) + cell.setCellValue(DataType.valueToString(elem).toDouble) + case _ => + val value = DataType.valueToString(elem) + cell.setCellValue(value) + } + } { case e: Exception => + cell.setCellValue(DataType.valueToString(elem)) + } + } + + private def getDate(value: Any): Date = { + if (value.isInstanceOf[Date]) { + value.asInstanceOf[Date] + } else { + throw new NumberFormatException( + s"Value ${value} with class : ${value.getClass.getName} is not a valid type of Date." + ); + } + } + + /** + * Check whether the double exceeds the number of digits, which will affect the data accuracy + * @param elemValue + */ + private def doubleCheck(elemValue: String): Unit = { + val value = new BigDecimal(elemValue).stripTrailingZeros + if ((value.precision - value.scale) > 15) { + throw new NumberFormatException( + s"Value ${elemValue} error : This data exceeds 15 significant digits." + ); + } + } + + override def flush(): Unit = { + getWorkBook.write(os) + val content: Array[Byte] = os.toByteArray + is = new ByteArrayInputStream(content) + val buffer: Array[Byte] = new Array[Byte](1024) + var bytesRead: Int = 0 + while (isFlush) { + bytesRead = is.read(buffer, 0, 1024) + if (bytesRead == -1) { + isFlush = false + } else { + outputStream.write(buffer, 0, bytesRead) + } + } + } + + override def close(): Unit = { + if (isFlush) flush() + IOUtils.closeQuietly(outputStream) + IOUtils.closeQuietly(is) + IOUtils.closeQuietly(os) + IOUtils.closeQuietly(workBook) + } + +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageMultiExcelWriter.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala similarity index 58% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageMultiExcelWriter.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala index a028397bd6..b47e04394f 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/StorageMultiExcelWriter.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageMultiExcelWriter.scala @@ -15,44 +15,39 @@ * limitations under the License. */ -package org.apache.linkis.storage.excel; +package org.apache.linkis.storage.excel -import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook -import java.io.OutputStream; +import java.io.OutputStream -public class StorageMultiExcelWriter extends StorageExcelWriter { +class StorageMultiExcelWriter( + override val outputStream: OutputStream, + override val autoFormat: Boolean +) extends StorageExcelWriter(null, null, null, outputStream, autoFormat) { - private int sheetIndex = 0; + private var sheetIndex = 0 - public StorageMultiExcelWriter(OutputStream outputStream, boolean autoFormat) { - super(null, null, null, outputStream, autoFormat); - } - - @Override - public void init() { - if (workBook == null) { - workBook = new SXSSFWorkbook(); - } + override def init: Unit = { + if (workBook == null) workBook = new SXSSFWorkbook() // 1.让表自适应列宽 if (sheet != null) { - sheet.trackAllColumnsForAutoSizing(); - for (int i = 0; i <= columnCounter; i++) { - sheet.autoSizeColumn(i); - } + sheet.trackAllColumnsForAutoSizing() + 0 to columnCounter foreach (sheet.autoSizeColumn) } // 2.重置参数 // 2.1 columnCounter 归0 - columnCounter = 0; + columnCounter = 0 // 2.2 创建新sheet - sheet = workBook.createSheet("resultset" + sheetIndex); + sheet = workBook.createSheet(s"resultset$sheetIndex") // 2.3 sheetIndex自增 - sheetIndex++; + sheetIndex += 1 // 2.4 types 置空 - types = null; + types = null // 2.5 rowPoint 归0 记录行数 - rowPoint = 0; + rowPoint = 0 // 2.6 styles 清空 - styles.clear(); + styles.clear() } + } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala new file mode 100644 index 0000000000..b21bf7e492 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOClient.scala @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.io + +import org.apache.linkis.storage.domain.MethodEntity +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.MUST_REGISTER_TOC +import org.apache.linkis.storage.exception.StorageErrorException + +import java.util.UUID + +import org.slf4j.{Logger, LoggerFactory} + +/** + * IOClient is used to execute the proxy as the ujes code execution entry in io and get the return + * result. IOClient用于在io进行代理作为ujes的代码执行入口，并获取返回结果 + */ +trait IOClient { + + def execute(user: String, methodEntity: MethodEntity, params: java.util.Map[String, Any]): String + + def executeWithEngine( + user: String, + methodEntity: MethodEntity, + params: java.util.Map[String, Any] + ): Array[String] + +} + +object IOClient { + val logger: Logger = LoggerFactory.getLogger(classOf[IOClient]) + var ioClient: IOClient = null + + val SUCCESS = "SUCCESS" + val FAILED = "FAILED" + + def getIOClient(): IOClient = { + if (ioClient == null) { + throw new StorageErrorException( + MUST_REGISTER_TOC.getErrorCode, + MUST_REGISTER_TOC.getErrorDesc + ) + } + ioClient + } + + /** + * This method is called when ioClient is initialized. ioClient初始化时会调用该方法 + * @param client + * IOClient + */ + def register(client: IOClient): Unit = { + this.ioClient = client + logger.debug(s"IOClient: ${ioClient.toString} registered") + } + + def getFSId(): String = { + UUID.randomUUID().toString + } + +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClientFactory.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala similarity index 53% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClientFactory.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala index 244ad59477..51e1589eb7 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/io/IOClientFactory.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/io/IOMethodInterceptorCreator.scala @@ -15,43 +15,39 @@ * limitations under the License. */ -package org.apache.linkis.storage.io; +package org.apache.linkis.storage.io -import org.apache.linkis.storage.exception.StorageErrorException; +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.MUST_REGISTER_TOM +import org.apache.linkis.storage.exception.StorageErrorException -import java.util.UUID; +import org.springframework.cglib.proxy.MethodInterceptor -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +trait IOMethodInterceptorCreator { -import static org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.MUST_REGISTER_TOC; - -public class IOClientFactory { - private static final Logger logger = LoggerFactory.getLogger(IOClientFactory.class); - private static IOClient ioClient = null; + def createIOMethodInterceptor(fsName: String): MethodInterceptor +} - private static final String SUCCESS = "SUCCESS"; - private static final String FAILED = "FAILED"; +object IOMethodInterceptorCreator { - public static IOClient getIOClient() throws StorageErrorException { - if (ioClient == null) { - throw new StorageErrorException( - MUST_REGISTER_TOC.getErrorCode(), MUST_REGISTER_TOC.getErrorDesc()); - } - return ioClient; - } + var interceptorCreator: IOMethodInterceptorCreator = null /** * This method is called when ioClient is initialized. ioClient初始化时会调用该方法 * - * @param client IOClient + * @param interceptorCreator */ - public static void register(IOClient client) { - ioClient = client; - logger.debug("IOClient: {} registered", ioClient.toString()); + def register(interceptorCreator: IOMethodInterceptorCreator): Unit = { + this.interceptorCreator = interceptorCreator } - public static String getFSId() { - return UUID.randomUUID().toString(); + def getIOMethodInterceptor(fsName: String): MethodInterceptor = { + if (interceptorCreator == null) { + throw new StorageErrorException( + MUST_REGISTER_TOM.getErrorCode, + MUST_REGISTER_TOM.getErrorDesc + ) + } + interceptorCreator.createIOMethodInterceptor(fsName) } + } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala new file mode 100644 index 0000000000..d4836731db --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.{Fs, FsPath, MetaData, Record} +import org.apache.linkis.common.io.resultset.ResultSet +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.domain.Dolphin +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.{ + THE_FILE_IS_EMPTY, + UNSUPPORTED_RESULT +} +import org.apache.linkis.storage.exception.{StorageErrorException, StorageWarnException} +import org.apache.linkis.storage.utils.{StorageConfiguration, StorageUtils} + +import org.apache.commons.lang3.StringUtils + +import java.text.MessageFormat +import java.util +import java.util.Locale + +class DefaultResultSetFactory extends ResultSetFactory with Logging { + + private val resultClasses: Map[String, Class[ResultSet[ResultMetaData, ResultRecord]]] = + StorageUtils.loadClasses( + StorageConfiguration.STORAGE_RESULT_SET_CLASSES.getValue, + StorageConfiguration.STORAGE_RESULT_SET_PACKAGE.getValue, + t => t.newInstance().resultSetType().toLowerCase(Locale.getDefault) + ) + + val resultTypes = ResultSetFactory.resultSetType.keys.toArray + + override def getResultSetByType(resultSetType: String): ResultSet[_ <: MetaData, _ <: Record] = { + if (!resultClasses.contains(resultSetType)) { + throw new StorageErrorException( + UNSUPPORTED_RESULT.getErrorCode, + MessageFormat.format(UNSUPPORTED_RESULT.getErrorDesc, resultSetType) + ) + } + resultClasses(resultSetType).newInstance() + } + + override def getResultSetByPath(fsPath: FsPath): ResultSet[_ <: MetaData, _ <: Record] = { + getResultSetByPath(fsPath, StorageUtils.getJvmUser) + } + + override def getResultSetByContent(content: String): ResultSet[_ <: MetaData, _ <: Record] = { + getResultSetByType(Dolphin.getType(content)) + } + + override def exists(resultSetType: String): Boolean = resultClasses.contains(resultSetType) + + override def isResultSetPath(path: String): Boolean = { + path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX) + } + + override def isResultSet(content: String): Boolean = + Utils.tryCatch(resultClasses.contains(Dolphin.getType(content))) { t => + logger.info("Wrong result Set: " + t.getMessage) + false + } + + override def getResultSet(output: String): ResultSet[_ <: MetaData, _ <: Record] = + getResultSet(output, StorageUtils.getJvmUser) + + override def getResultSetType: Array[String] = resultTypes + + override def getResultSetByPath(fsPath: FsPath, fs: Fs): ResultSet[_ <: MetaData, _ <: Record] = { + val inputStream = fs.read(fsPath) + val resultSetType = Dolphin.getType(inputStream) + if (StringUtils.isEmpty(resultSetType)) { + throw new StorageWarnException( + THE_FILE_IS_EMPTY.getErrorCode, + MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc, fsPath.getPath) + ) + } + Utils.tryQuietly(inputStream.close()) + // Utils.tryQuietly(fs.close()) + getResultSetByType(resultSetType) + } + + override def getResultSetByPath( + fsPath: FsPath, + proxyUser: String + ): ResultSet[_ <: MetaData, _ <: Record] = { + if (fsPath == null) return null + logger.info("Get Result Set By Path:" + fsPath.getPath) + val fs = FSFactory.getFsByProxyUser(fsPath, proxyUser) + fs.init(new util.HashMap[String, String]()) + val inputStream = fs.read(fsPath) + val resultSetType = Dolphin.getType(inputStream) + if (StringUtils.isEmpty(resultSetType)) { + throw new StorageWarnException( + THE_FILE_IS_EMPTY.getErrorCode, + MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc, fsPath.getPath) + ) + } + Utils.tryQuietly(inputStream.close()) + Utils.tryQuietly(fs.close()) + getResultSetByType(resultSetType) + } + + override def getResultSet( + output: String, + proxyUser: String + ): ResultSet[_ <: MetaData, _ <: Record] = { + if (isResultSetPath(output)) { + getResultSetByPath(new FsPath(output), proxyUser) + } else if (isResultSet(output)) { + getResultSetByContent(output) + } else null + } + +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultRecord.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultMetaData.scala similarity index 85% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultRecord.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultMetaData.scala index ce2686099b..7871bb2ac9 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultRecord.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultMetaData.scala @@ -15,8 +15,8 @@ * limitations under the License. */ -package org.apache.linkis.storage.resultset; +package org.apache.linkis.storage.resultset -import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.MetaData -public interface ResultRecord extends Record {} +abstract class ResultMetaData extends MetaData {} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultRecord.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultRecord.scala new file mode 100644 index 0000000000..ab9244ca98 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultRecord.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.Record + +abstract class ResultRecord extends Record {} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetFactory.scala new file mode 100644 index 0000000000..e6615873b1 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetFactory.scala @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.{Fs, FsPath, MetaData, Record} +import org.apache.linkis.common.io.resultset.ResultSet + +import scala.collection.mutable + +trait ResultSetFactory extends scala.AnyRef { + + def getResultSetByType(resultSetType: scala.Predef.String): ResultSet[_ <: MetaData, _ <: Record] + + def getResultSetByPath(fsPath: FsPath): ResultSet[_ <: MetaData, _ <: Record] + def getResultSetByPath(fsPath: FsPath, fs: Fs): ResultSet[_ <: MetaData, _ <: Record] + def getResultSetByContent(content: scala.Predef.String): ResultSet[_ <: MetaData, _ <: Record] + def exists(resultSetType: scala.Predef.String): scala.Boolean + def isResultSetPath(path: scala.Predef.String): scala.Boolean + def isResultSet(content: scala.Predef.String): scala.Boolean + def getResultSet(output: String): ResultSet[_ <: MetaData, _ <: Record] + + def getResultSetByPath(fsPath: FsPath, proxyUser: String): ResultSet[_ <: MetaData, _ <: Record] + + def getResultSet(output: String, proxyUser: String): ResultSet[_ <: MetaData, _ <: Record] + + /** + * The first must-time text(第一个必须时text) + * @return + */ + def getResultSetType: Array[String] +} + +object ResultSetFactory { + + val TEXT_TYPE = "1" + val TABLE_TYPE = "2" + val IO_TYPE = "3" + val PICTURE_TYPE = "4" + val HTML_TYPE = "5" + + /** + * TODO 修改为注册形式，并修改ResultSet的getResultType逻辑 Result set corresponding type record(结果集对应类型记录) + */ + val resultSetType = mutable.LinkedHashMap[String, String]( + TEXT_TYPE -> "TEXT", + TABLE_TYPE -> "TABLE", + IO_TYPE -> "IO", + PICTURE_TYPE -> "PICTURE", + HTML_TYPE -> "HTML" + ) + + val factory = new DefaultResultSetFactory + def getInstance: ResultSetFactory = factory +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala new file mode 100644 index 0000000000..e61cf36b3d --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.{FsPath, MetaData, Record} +import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader} +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED +import org.apache.linkis.storage.exception.StorageErrorException +import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord, TableResultSet} + +import java.io.InputStream + +object ResultSetReader extends Logging { + + def getResultSetReader[K <: MetaData, V <: Record]( + resultSet: ResultSet[K, V], + inputStream: InputStream + ): ResultSetReader[K, V] = { + new StorageResultSetReader[K, V](resultSet, inputStream) + } + + def getResultSetReader[K <: MetaData, V <: Record]( + resultSet: ResultSet[K, V], + value: String + ): ResultSetReader[K, V] = { + new StorageResultSetReader[K, V](resultSet, value) + } + + def getResultSetReader(res: String): ResultSetReader[_ <: MetaData, _ <: Record] = { + val rsFactory = ResultSetFactory.getInstance + if (rsFactory.isResultSet(res)) { + val resultSet = rsFactory.getResultSet(res) + ResultSetReader.getResultSetReader(resultSet, res) + } else { + val resPath = new FsPath(res) + val resultSet = rsFactory.getResultSetByPath(resPath) + val fs = FSFactory.getFs(resPath) + fs.init(null) + val reader = ResultSetReader.getResultSetReader(resultSet, fs.read(resPath)) + reader match { + case storageResultSetReader: StorageResultSetReader[_, _] => + storageResultSetReader.setFs(fs) + case _ => + } + reader + } + } + + def getTableResultReader(res: String): ResultSetReader[TableMetaData, TableRecord] = { + val rsFactory = ResultSetFactory.getInstance + if (rsFactory.isResultSet(res)) { + val resultSet = rsFactory.getResultSet(res) + if (ResultSetFactory.TABLE_TYPE != resultSet.resultSetType()) { + throw new StorageErrorException( + TABLE_ARE_NOT_SUPPORTED.getErrorCode, + TABLE_ARE_NOT_SUPPORTED.getErrorDesc + ) + } + ResultSetReader.getResultSetReader(resultSet.asInstanceOf[TableResultSet], res) + } else { + val resPath = new FsPath(res) + val resultSet = rsFactory.getResultSetByPath(resPath) + if (ResultSetFactory.TABLE_TYPE != resultSet.resultSetType()) { + throw new StorageErrorException( + TABLE_ARE_NOT_SUPPORTED.getErrorCode, + TABLE_ARE_NOT_SUPPORTED.getErrorDesc + ) + } + val fs = FSFactory.getFs(resPath) + logger.info("Try to init Fs with path:" + resPath.getPath) + fs.init(null) + ResultSetReader.getResultSetReader(resultSet.asInstanceOf[TableResultSet], fs.read(resPath)) + } + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetWriter.scala new file mode 100644 index 0000000000..cea00bbd60 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetWriter.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.{FsPath, MetaData, Record} +import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetWriter} + +import scala.collection.mutable.ArrayBuffer + +object ResultSetWriter { + + def getResultSetWriter[K <: MetaData, V <: Record]( + resultSet: ResultSet[K, V], + maxCacheSize: Long, + storePath: FsPath + ): ResultSetWriter[K, V] = + new StorageResultSetWriter[K, V](resultSet, maxCacheSize, storePath) + + def getResultSetWriter[K <: MetaData, V <: Record]( + resultSet: ResultSet[K, V], + maxCacheSize: Long, + storePath: FsPath, + proxyUser: String + ): ResultSetWriter[K, V] = { + val writer = new StorageResultSetWriter[K, V](resultSet, maxCacheSize, storePath) + writer.setProxyUser(proxyUser) + writer + } + + def getRecordByWriter( + writer: ResultSetWriter[_ <: MetaData, _ <: Record], + limit: Long + ): Array[Record] = { + val res = writer.toString + getRecordByRes(res, limit) + } + + def getRecordByRes(res: String, limit: Long): Array[Record] = { + val reader = ResultSetReader.getResultSetReader(res) + var count = 0 + val records = new ArrayBuffer[Record]() + reader.getMetaData + while (reader.hasNext && count < limit) { + records += reader.getRecord + count = count + 1 + } + records.toArray + } + + def getLastRecordByRes(res: String): Record = { + val reader = ResultSetReader.getResultSetReader(res) + reader.getMetaData + while (reader.hasNext) { + reader.getRecord + } + reader.getRecord + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala new file mode 100644 index 0000000000..7b3aca62d9 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.{FsPath, MetaData, Record} +import org.apache.linkis.common.io.resultset.ResultSet +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.domain.Dolphin +import org.apache.linkis.storage.utils.StorageConfiguration + +abstract class StorageResultSet[K <: MetaData, V <: Record] extends ResultSet[K, V] with Logging { + + val resultHeaderBytes = Dolphin.MAGIC_BYTES ++ Dolphin.getIntBytes(resultSetType().toInt) + override val charset: String = StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue + + override def getResultSetPath(parentDir: FsPath, fileName: String): FsPath = { + val path = if (parentDir.getPath.endsWith("/")) { + parentDir.toPath + fileName + Dolphin.DOLPHIN_FILE_SUFFIX + } else { + parentDir.toPath + "/" + fileName + Dolphin.DOLPHIN_FILE_SUFFIX + } + logger.info(s"Get result set path:${path}") + new FsPath(path) + } + + override def getResultSetHeader: Array[Byte] = resultHeaderBytes + + override def belongToPath(path: String): Boolean = path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX) + + override def belongToResultSet(content: String): Boolean = + Utils.tryCatch(Dolphin.getType(content) == resultSetType()) { t => + logger.info("Wrong result Set: ", t) + false + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala new file mode 100644 index 0000000000..24dd0fdf8a --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.{Fs, MetaData, Record} +import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader} +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.conf.LinkisStorageConf +import org.apache.linkis.storage.domain.Dolphin +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary +import org.apache.linkis.storage.exception.{ + ColLengthExceedException, + StorageErrorCode, + StorageErrorException, + StorageWarnException +} +import org.apache.linkis.storage.resultset.table.TableMetaData +import org.apache.linkis.storage.utils.StorageUtils + +import java.io.{ByteArrayInputStream, InputStream, IOException} +import java.text.MessageFormat + +import scala.collection.mutable.ArrayBuffer + +class StorageResultSetReader[K <: MetaData, V <: Record]( + resultSet: ResultSet[K, V], + inputStream: InputStream +) extends ResultSetReader[K, V](resultSet, inputStream) + with Logging { + + private val deserializer = resultSet.createResultSetDeserializer + private var metaData: K = _ + private var row: Record = _ + private var rowCount = 0 + + private var fs: Fs = _ + + def this(resultSet: ResultSet[K, V], value: String) = { + this(resultSet, new ByteArrayInputStream(value.getBytes(Dolphin.CHAR_SET))) + } + + def init(): Unit = { + val resType = Dolphin.getType(inputStream) + if (resultSet.resultSetType != resType) { + throw new IOException( + "File type does not match(文件类型不匹配): " + ResultSetFactory.resultSetType + .getOrElse(resType, "TABLE") + ) + } + } + + /** + * Read a row of data Read the line length first Get the entire row of data by the length of the + * line, first obtain the column length in the entire row of data, and then divide into column + * length to split the data 读取一行数据 先读取行长度 通过行长度获取整行数据，在整行数据中先获取列长度，进而分割成列长度从而分割数据 + * @return + */ + def readLine(): Array[Byte] = { + + var rowLen = 0 + try rowLen = Dolphin.readInt(inputStream) + catch { + case _: StorageWarnException => logger.info(s"Read finished(读取完毕)"); return null + case t: Throwable => throw t + } + + if (rowLen > LinkisStorageConf.LINKIS_READ_ROW_BYTE_MAX_LEN) { + throw new ColLengthExceedException( + LinkisStorageErrorCodeSummary.RESULT_ROW_LENGTH.getErrorCode, + MessageFormat.format( + LinkisStorageErrorCodeSummary.RESULT_ROW_LENGTH.getErrorDesc, + rowLen.asInstanceOf[Object], + LinkisStorageConf.LINKIS_READ_ROW_BYTE_MAX_LEN.asInstanceOf[Object] + ) + ) + } + + var bytes: Array[Byte] = null + try { + bytes = new Array[Byte](rowLen) + } catch { + case e: OutOfMemoryError => + logger.error("Result set read oom, read size {} Byte", rowLen) + throw new StorageErrorException( + StorageErrorCode.FS_OOM.getCode, + StorageErrorCode.FS_OOM.getMessage, + e + ) + } + val len = StorageUtils.readBytes(inputStream, bytes, rowLen) + if (len != rowLen) { + throw new StorageErrorException( + StorageErrorCode.INCONSISTENT_DATA.getCode, + String.format(StorageErrorCode.INCONSISTENT_DATA.getMessage, len.toString, rowLen.toString) + ) + } + rowCount = rowCount + 1 + bytes + } + + @scala.throws[IOException] + override def getRecord: Record = { + if (metaData == null) throw new IOException("Must read metadata first(必须先读取metadata)") + if (row == null) { + throw new IOException( + "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)" + ) + } + row + } + + def setFs(fs: Fs): Unit = this.fs = fs + def getFs: Fs = this.fs + + @scala.throws[IOException] + override def getMetaData: MetaData = { + if (metaData == null) init() + metaData = deserializer.createMetaData(readLine()) + metaData + } + + @scala.throws[IOException] + override def skip(recordNum: Int): Int = { + if (recordNum < 0) return -1 + + if (metaData == null) getMetaData + for (i <- recordNum until (0, -1)) { + try inputStream.skip(Dolphin.readInt(inputStream)) + catch { + case t: Throwable => + return recordNum - i + } + } + recordNum + } + + @scala.throws[IOException] + override def getPosition: Long = rowCount + + @scala.throws[IOException] + override def hasNext: Boolean = { + if (metaData == null) getMetaData + + val line = readLine() + if (line == null) return false + row = deserializer.createRecord(line) + if (row == null) return false + true + } + + @scala.throws[IOException] + override def available: Long = inputStream.available() + + override def close(): Unit = { + inputStream.close() + if (this.fs != null) Utils.tryQuietly(this.fs.close()) + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala new file mode 100644 index 0000000000..caed8c0ea0 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset + +import org.apache.linkis.common.io.{Fs, FsPath, MetaData, Record} +import org.apache.linkis.common.io.resultset.{ResultSerializer, ResultSet, ResultSetWriter} +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.conf.LinkisStorageConf +import org.apache.linkis.storage.domain.Dolphin +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.fs.impl.HDFSFileSystem +import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils} + +import org.apache.commons.io.IOUtils +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream + +import java.io.{IOException, OutputStream} + +import scala.collection.mutable.ArrayBuffer + +class StorageResultSetWriter[K <: MetaData, V <: Record]( + resultSet: ResultSet[K, V], + maxCacheSize: Long, + storePath: FsPath +) extends ResultSetWriter[K, V]( + resultSet = resultSet, + maxCacheSize = maxCacheSize, + storePath = storePath + ) + with Logging { + + private val serializer: ResultSerializer = resultSet.createResultSetSerializer + + private var moveToWriteRow = false + + private var outputStream: OutputStream = _ + + private var rowCount = 0 + + private val buffer = new ArrayBuffer[Byte]() + + private var fs: Fs = _ + + private var rMetaData: MetaData = _ + + private var proxyUser: String = StorageUtils.getJvmUser + + private var fileCreated = false + + private var closed = false + + private val WRITER_LOCK_CREATE = new Object() + + private val WRITER_LOCK_CLOSE = new Object() + + def getMetaData: MetaData = rMetaData + + def setProxyUser(proxyUser: String): Unit = { + this.proxyUser = proxyUser + } + + def isEmpty: Boolean = { + rMetaData == null && buffer.length <= Dolphin.FILE_EMPTY + } + + def init(): Unit = { + writeLine(resultSet.getResultSetHeader, true) + } + + def createNewFile: Unit = { + if (!fileCreated) { + WRITER_LOCK_CREATE.synchronized { + if (!fileCreated) { + if (storePath != null && outputStream == null) { + logger.info(s"Try to create a new file:${storePath}, with proxy user:${proxyUser}") + fs = FSFactory.getFsByProxyUser(storePath, proxyUser) + fs.init(null) + FileSystemUtils.createNewFile(storePath, proxyUser, true) + + outputStream = fs.write(storePath, true) + fs match { + case fileSystem: FileSystem => + fileSystem.setPermission(storePath, "rwx------") + case _ => + } + logger.info(s"Succeed to create a new file:$storePath") + fileCreated = true + } + } + } + } else if (null != storePath && null == outputStream) { + logger.warn("outputStream had been set null, but createNewFile() was called again.") + } + } + + def writeLine(bytes: Array[Byte], cache: Boolean = false): Unit = { + if (closed) { + logger.warn("the writer had been closed, but writeLine() was still called.") + return + } + if (bytes.length > LinkisStorageConf.ROW_BYTE_MAX_LEN) { + throw new IOException( + s"A single row of data cannot exceed ${LinkisStorageConf.ROW_BYTE_MAX_LEN_STR}" + ) + } + if (buffer.length > maxCacheSize && !cache) { + if (outputStream == null) { + createNewFile + } + flush() + outputStream.write(bytes) + } else { + buffer.appendAll(bytes) + } + } + + override def toString: String = { + if (outputStream == null) { + if (isEmpty) return "" + new String(buffer.toArray, Dolphin.CHAR_SET) + } else { + storePath.getSchemaPath + } + } + + override def toFSPath: FsPath = storePath + + override def addMetaDataAndRecordString(content: String): Unit = { + if (!moveToWriteRow) { + val bytes = content.getBytes(Dolphin.CHAR_SET) + writeLine(bytes) + } + moveToWriteRow = true + } + + override def addRecordString(content: String): Unit = {} + + @scala.throws[IOException] + override def addMetaData(metaData: MetaData): Unit = { + if (!moveToWriteRow) { + { + rMetaData = metaData + init() + if (null == metaData) { + writeLine(serializer.metaDataToBytes(metaData), true) + } else { + writeLine(serializer.metaDataToBytes(metaData)) + } + } + moveToWriteRow = true + } + } + + @scala.throws[IOException] + override def addRecord(record: Record): Unit = { + if (moveToWriteRow) { + rowCount = rowCount + 1 + writeLine(serializer.recordToBytes(record)) + } + } + + def closeFs: Unit = { + if (fs != null) { + IOUtils.closeQuietly(fs) + fs = null + } + } + + override def close(): Unit = { + if (closed) { + logger.warn("the writer had been closed, but close() was still called.") + return + } else { + WRITER_LOCK_CLOSE.synchronized { + if (!closed) { + closed = true + } else { + return + } + } + } + Utils.tryFinally(if (outputStream != null) flush()) { + if (outputStream != null) { + IOUtils.closeQuietly(outputStream) + outputStream = null + } + closeFs + } + } + + override def flush(): Unit = { + createNewFile + if (outputStream != null) { + if (buffer.nonEmpty) { + outputStream.write(buffer.toArray) + buffer.clear() + } + Utils.tryAndWarnMsg[Unit] { + outputStream match { + case hdfs: HdfsDataOutputStream => + hdfs.hflush() + case _ => + outputStream.flush() + } + }(s"Error encounters when flush result set ") + } + if (closed) { + if (logger.isDebugEnabled()) { + logger.debug("the writer had been closed, but flush() was still called.") + } + } + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/html/HtmlResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/html/HtmlResultSet.scala new file mode 100644 index 0000000000..7a65a9b9d5 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/html/HtmlResultSet.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.html + +import org.apache.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} +import org.apache.linkis.storage.{LineMetaData, LineRecord} +import org.apache.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} +import org.apache.linkis.storage.resultset.txt.{TextResultDeserializer, TextResultSerializer} + +class HtmlResultSet extends StorageResultSet[LineMetaData, LineRecord] with Serializable { + + override def resultSetType(): String = ResultSetFactory.HTML_TYPE + + override def createResultSetSerializer(): ResultSerializer = new TextResultSerializer + + override def createResultSetDeserializer(): ResultDeserializer[LineMetaData, LineRecord] = + new TextResultDeserializer + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOMetaData.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOMetaData.scala new file mode 100644 index 0000000000..32b578c5a8 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOMetaData.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.io + +import org.apache.linkis.common.io.MetaData +import org.apache.linkis.storage.resultset.ResultMetaData + +class IOMetaData(val off: Int, val len: Int) extends ResultMetaData { + override def cloneMeta(): MetaData = new IOMetaData(off, len) +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IORecord.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IORecord.scala new file mode 100644 index 0000000000..54d3dc0224 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IORecord.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.io + +import org.apache.linkis.common.io.Record +import org.apache.linkis.storage.resultset.ResultRecord + +class IORecord(val value: Array[Byte]) extends ResultRecord { + override def cloneRecord(): Record = new IORecord(value) +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsReader.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultDeserializer.scala similarity index 56% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsReader.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultDeserializer.scala index 00b79eb7e4..4e8199da04 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/ScriptFsReader.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultDeserializer.scala @@ -15,26 +15,22 @@ * limitations under the License. */ -package org.apache.linkis.storage.script; +package org.apache.linkis.storage.resultset.io -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.common.io.FsReader; -import org.apache.linkis.storage.script.reader.StorageScriptFsReader; +import org.apache.linkis.common.io.resultset.ResultDeserializer +import org.apache.linkis.storage.domain.Dolphin -import java.io.InputStream; +import org.apache.commons.codec.binary.Base64 -public abstract class ScriptFsReader extends FsReader { +class IOResultDeserializer extends ResultDeserializer[IOMetaData, IORecord] { - protected FsPath path; - protected String charset; - - public ScriptFsReader(FsPath path, String charset) { - this.path = path; - this.charset = charset; + override def createMetaData(bytes: Array[Byte]): IOMetaData = { + val values = Dolphin.getString(bytes, 0, bytes.length).split(Dolphin.COL_SPLIT) + new IOMetaData(values(0).toInt, values(1).toInt) } - public static ScriptFsReader getScriptFsReader( - FsPath path, String charset, InputStream inputStream) { - return new StorageScriptFsReader(path, charset, inputStream); + override def createRecord(bytes: Array[Byte]): IORecord = { + new IORecord(Base64.decodeBase64(Dolphin.getString(bytes, 0, bytes.length))) } + } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSerializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSerializer.scala new file mode 100644 index 0000000000..be527e3969 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSerializer.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.io + +import org.apache.linkis.common.io.{MetaData, Record} +import org.apache.linkis.common.io.resultset.ResultSerializer +import org.apache.linkis.storage.domain.Dolphin + +import org.apache.commons.codec.binary.Base64 + +class IOResultSerializer extends ResultSerializer { + + override def metaDataToBytes(metaData: MetaData): Array[Byte] = { + val ioMetaData = metaData.asInstanceOf[IOMetaData] + lineToBytes(s"${ioMetaData.off}${Dolphin.COL_SPLIT}${ioMetaData.len}") + } + + override def recordToBytes(record: Record): Array[Byte] = { + val ioRecord = record.asInstanceOf[IORecord] + lineToBytes(Base64.encodeBase64String(ioRecord.value)) + } + + def lineToBytes(value: String): Array[Byte] = { + val bytes = if (value == null) Dolphin.NULL_BYTES else Dolphin.getBytes(value) + Dolphin.getIntBytes(bytes.length) ++ bytes + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSet.scala new file mode 100644 index 0000000000..5fd952dbf9 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/io/IOResultSet.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.io + +import org.apache.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} +import org.apache.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} + +class IOResultSet extends StorageResultSet[IOMetaData, IORecord] with Serializable { + + override def resultSetType(): String = ResultSetFactory.IO_TYPE + + override def createResultSetSerializer(): ResultSerializer = new IOResultSerializer + + override def createResultSetDeserializer(): ResultDeserializer[IOMetaData, IORecord] = + new IOResultDeserializer + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/label/LabelUpdateRequest.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/picture/PictureResultSet.scala similarity index 53% rename from linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/label/LabelUpdateRequest.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/picture/PictureResultSet.scala index 57d05200ae..95c1d3ad1f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/label/LabelUpdateRequest.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/picture/PictureResultSet.scala @@ -15,22 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.manager.common.protocol.label; +package org.apache.linkis.storage.resultset.picture -import org.apache.linkis.manager.label.entity.Label; -import org.apache.linkis.protocol.message.RequestProtocol; +import org.apache.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} +import org.apache.linkis.storage.{LineMetaData, LineRecord} +import org.apache.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} +import org.apache.linkis.storage.resultset.txt.{TextResultDeserializer, TextResultSerializer} -import java.util.List; +class PictureResultSet extends StorageResultSet[LineMetaData, LineRecord] with Serializable { -public class LabelUpdateRequest implements RequestProtocol { + override def resultSetType(): String = ResultSetFactory.PICTURE_TYPE - private final List> labels; + override def createResultSetSerializer(): ResultSerializer = new TextResultSerializer - public LabelUpdateRequest(List> labels) { - this.labels = labels; - } + override def createResultSetDeserializer(): ResultDeserializer[LineMetaData, LineRecord] = + new TextResultDeserializer - public List> getLabels() { - return labels; - } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/RequestEngineUnlock.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableMetaData.scala similarity index 70% rename from linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/RequestEngineUnlock.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableMetaData.scala index 9646633e3d..526078d6c9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/RequestEngineUnlock.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableMetaData.scala @@ -15,22 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.manager.common.protocol; +package org.apache.linkis.storage.resultset.table -public class RequestEngineUnlock implements EngineLock { - private String lock; +import org.apache.linkis.common.io.MetaData +import org.apache.linkis.storage.domain.Column +import org.apache.linkis.storage.resultset.ResultMetaData - public RequestEngineUnlock() {} +class TableMetaData(val columns: Array[Column]) extends ResultMetaData { - public RequestEngineUnlock(String lock) { - this.lock = lock; + override def cloneMeta(): MetaData = { + new TableMetaData(columns) } - public String getLock() { - return lock; - } - - public void setLock(String lock) { - this.lock = lock; - } } diff --git a/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/PythonExecuteError.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala similarity index 71% rename from linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/PythonExecuteError.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala index 8a45ea078d..64d2b18b9c 100644 --- a/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/exception/PythonExecuteError.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableRecord.scala @@ -15,12 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.python.exception; +package org.apache.linkis.storage.resultset.table -import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.io.Record +import org.apache.linkis.storage.resultset.ResultRecord +import org.apache.linkis.storage.utils.StorageUtils -public class PythonExecuteError extends ErrorException { - public PythonExecuteError(int errCode, String desc) { - super(errCode, desc); +class TableRecord(val row: Array[Any]) extends ResultRecord { + + override def cloneRecord(): Record = { + new TableRecord(row) } + } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala new file mode 100644 index 0000000000..86d09e9532 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultDeserializer.scala @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.table + +import org.apache.linkis.common.io.resultset.ResultDeserializer +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.storage.conf.LinkisStorageConf +import org.apache.linkis.storage.domain.{Column, DataType, Dolphin} +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.PARSING_METADATA_FAILED +import org.apache.linkis.storage.exception.{ + ColLengthExceedException, + ColumnIndexExceedException, + StorageErrorException +} + +import org.apache.commons.lang3.StringUtils + +import java.text.MessageFormat + +import scala.collection.mutable.ArrayBuffer + +class TableResultDeserializer extends ResultDeserializer[TableMetaData, TableRecord] with Logging { + + var metaData: TableMetaData = _ + + var columnSet: Set[Int] = null + + import DataType._ + + override def createMetaData(bytes: Array[Byte]): TableMetaData = { + val colByteLen = Dolphin.getString(bytes, 0, Dolphin.INT_LEN).toInt + val colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen) + val colArray = + if (colString.endsWith(Dolphin.COL_SPLIT)) { + colString.substring(0, colString.length - 1).split(Dolphin.COL_SPLIT) + } else colString.split(Dolphin.COL_SPLIT) + var index = Dolphin.INT_LEN + colByteLen + if (colArray.length % 3 != 0) { + throw new StorageErrorException( + PARSING_METADATA_FAILED.getErrorCode, + PARSING_METADATA_FAILED.getErrorDesc + ) + } + val columns = new ArrayBuffer[Column]() + for (i <- 0 until (colArray.length, 3)) { + var len = colArray(i).toInt + val colName = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)) + index += len + len = colArray(i + 1).toInt + val colType = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)) + index += len + len = colArray(i + 2).toInt + val colComment = Dolphin.toStringValue(Dolphin.getString(bytes, index, len)) + index += len + columns += Column(colName, colType, colComment) + } + metaData = new TableMetaData(columns.toArray) + metaData + } + + /** + * colByteLen:All column fields are long(所有列字段长 记录的长度) colString：Obtain column + * length(获得列长)：10，20，21 colArray：Column length array(列长数组) Get data by column length(通过列长获得数据) + * @param bytes + * @return + */ + override def createRecord(bytes: Array[Byte]): TableRecord = { + val colByteLen = Dolphin.getString(bytes, 0, Dolphin.INT_LEN).toInt + val colString = Dolphin.getString(bytes, Dolphin.INT_LEN, colByteLen) + val colArray = + if (colString.endsWith(Dolphin.COL_SPLIT)) { + colString.substring(0, colString.length - 1).split(Dolphin.COL_SPLIT) + } else colString.split(Dolphin.COL_SPLIT) + var index = Dolphin.INT_LEN + colByteLen + var enableLimit: Boolean = false + if (StringUtils.isNotBlank(LinkisStorageConf.enableLimitThreadLocal.get())) { + enableLimit = true + } + val columnIndices: Array[Int] = LinkisStorageConf.columnIndicesThreadLocal.get() + if (columnSet == null && columnIndices != null) { + columnSet = columnIndices.toSet + } + + val lastIndex = + if (columnIndices != null && columnIndices.length > 0) columnIndices(columnIndices.length - 1) + else 0 + var realValueSize = colArray.size + + if (enableLimit && metaData.columns.size <= columnIndices(0)) { + throw new ColumnIndexExceedException( + LinkisStorageErrorCodeSummary.RESULT_COLUMN_INDEX_OUT_OF_BOUNDS.getErrorCode, + MessageFormat.format( + LinkisStorageErrorCodeSummary.RESULT_COLUMN_INDEX_OUT_OF_BOUNDS.getErrorDesc, + columnIndices(0).asInstanceOf[Object], + metaData.columns.size.asInstanceOf[Object] + ) + ) + } + + if (enableLimit && metaData.columns.size > lastIndex) { + realValueSize = columnIndices.length + } else if (enableLimit && metaData.columns.size <= lastIndex) { + realValueSize = metaData.columns.size % columnIndices.length + } + + val columnSize = colArray.size + val rowArray = new Array[Any](realValueSize) + + var colIdx = 0 + for (i <- 0 until columnSize) { + val len = colArray(i).toInt + val res = Dolphin.getString(bytes, index, len) + if (res.length > LinkisStorageConf.LINKIS_RESULT_COL_LENGTH && enableLimit) { + throw new ColLengthExceedException( + LinkisStorageErrorCodeSummary.RESULT_COL_LENGTH.getErrorCode, + MessageFormat.format( + LinkisStorageErrorCodeSummary.RESULT_COL_LENGTH.getErrorDesc, + res.length.asInstanceOf[Object], + LinkisStorageConf.LINKIS_RESULT_COL_LENGTH.asInstanceOf[Object] + ) + ) + } + index += len + // 如果enableLimit为true，则采取的是列分页 + if (enableLimit) { + if (columnSet.contains(i)) { + rowArray(colIdx) = toValue(metaData.columns(i).dataType, res) + colIdx += 1 + } + } else { + if (i >= metaData.columns.length) rowArray(i) = res + else { + rowArray(i) = toValue(metaData.columns(i).dataType, res) + } + } + } + new TableRecord(rowArray) + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala new file mode 100644 index 0000000000..5d1738a346 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSerializer.scala @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.table + +import org.apache.linkis.common.io.{MetaData, Record} +import org.apache.linkis.common.io.resultset.ResultSerializer +import org.apache.linkis.storage.conf.LinkisStorageConf +import org.apache.linkis.storage.domain.Dolphin + +import scala.collection.mutable.ArrayBuffer + +class TableResultSerializer extends ResultSerializer { + + override def metaDataToBytes(metaData: MetaData): Array[Byte] = { + val tableMetaData = metaData.asInstanceOf[TableMetaData] + lineToBytes(tableMetaData.columns.map(_.toArray).reduce((a1, a2) => a1 ++ a2)) + } + + override def recordToBytes(record: Record): Array[Byte] = { + val tableRecord = record.asInstanceOf[TableRecord] + lineToBytes(tableRecord.row) + } + + /** + * Convert a row of data to an array of Bytes Convert the data to byte and get the corresponding + * total byte length to write to the file Data write format: line length (fixed length) column + * length (fixed length) field index comma segmentation real data For example: + * 000000004900000000116,10,3,4,5,peace1johnnwang1101true11.51 The length of the line does not + * include its own length 将一行数据转换为Bytes的数组 对数据转换为byte，并获取相应的总byte长度写入文件 数据写入格式：行长(固定长度) 列长(固定长度) + * 字段索引逗号分割 真实数据 如：000000004900000000116,10,3,4,5,peace1johnnwang1101true11.51 其中行长不包括自身长度 + * @param line + */ + def lineToBytes(line: Array[Any]): Array[Byte] = { + val dataBytes = ArrayBuffer[Array[Byte]]() + val colIndex = ArrayBuffer[Array[Byte]]() + var colByteLen = 0 + var length = 0 + line.foreach { data => + val bytes = if (data == null) { + if (!LinkisStorageConf.LINKIS_RESULT_ENABLE_NULL) { + Dolphin.LINKIS_NULL_BYTES + } else { + Dolphin.NULL_BYTES + } + } else { + Dolphin.getBytes(data) + } + dataBytes += bytes + val colBytes = Dolphin.getBytes(bytes.length) + colIndex += colBytes += Dolphin.COL_SPLIT_BYTES + colByteLen += colBytes.length + Dolphin.COL_SPLIT_LEN + length += bytes.length + } + length += colByteLen + Dolphin.INT_LEN + toByteArray(length, colByteLen, colIndex, dataBytes) + } + + /** + * Splice a row of data into a byte array(将一行的数据拼接成byte数组) + * @param length + * The total length of the line data byte, excluding its own length(行数据byte总长度，不包括自身的长度) + * @param colByteLen + * Record field index byte column length(记录字段索引byte的列长) + * @param colIndex + * Field index, including separator comma(字段索引，包括分割符逗号) + * @param dataBytes + * Byte of real data(真实数据的byte) + * @return + */ + def toByteArray( + length: Int, + colByteLen: Int, + colIndex: ArrayBuffer[Array[Byte]], + dataBytes: ArrayBuffer[Array[Byte]] + ): Array[Byte] = { + val row = ArrayBuffer[Byte]() + colIndex ++= dataBytes + row.appendAll(Dolphin.getIntBytes(length)) + row.appendAll(Dolphin.getIntBytes(colByteLen)) + colIndex.foreach(row.appendAll(_)) + row.toArray + } + +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableMetaData.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSet.scala similarity index 57% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableMetaData.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSet.scala index 429ab33c82..fe8c4e9cd1 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/table/TableMetaData.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/table/TableResultSet.scala @@ -15,30 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.storage.resultset.table; +package org.apache.linkis.storage.resultset.table -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.storage.domain.Column; -import org.apache.linkis.storage.resultset.ResultMetaData; +import org.apache.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} +import org.apache.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} -public class TableMetaData implements ResultMetaData { +class TableResultSet extends StorageResultSet[TableMetaData, TableRecord] with Serializable { - public Column[] columns; + override def resultSetType(): String = ResultSetFactory.TABLE_TYPE - public TableMetaData(Column[] columns) { - this.columns = columns; - } + override def createResultSetSerializer(): ResultSerializer = new TableResultSerializer - public Column[] getColumns() { - return columns; - } + override def createResultSetDeserializer(): ResultDeserializer[TableMetaData, TableRecord] = + new TableResultDeserializer - public void setColumns(Column[] columns) { - this.columns = columns; - } - - @Override - public MetaData cloneMeta() { - return new TableMetaData(columns.clone()); - } } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.scala new file mode 100644 index 0000000000..b306b1f29a --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultDeserializer.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.txt + +import org.apache.linkis.common.io.resultset.ResultDeserializer +import org.apache.linkis.storage.{LineMetaData, LineRecord} +import org.apache.linkis.storage.domain.Dolphin + +class TextResultDeserializer extends ResultDeserializer[LineMetaData, LineRecord] { + + override def createMetaData(bytes: Array[Byte]): LineMetaData = { + new LineMetaData(Dolphin.getString(bytes, 0, bytes.length)) + } + + override def createRecord(bytes: Array[Byte]): LineRecord = { + new LineRecord(Dolphin.getString(bytes, 0, bytes.length)) + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSerializer.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSerializer.scala new file mode 100644 index 0000000000..e6e53338d0 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSerializer.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.txt + +import org.apache.linkis.common.io.{MetaData, Record} +import org.apache.linkis.common.io.resultset.ResultSerializer +import org.apache.linkis.storage.{LineMetaData, LineRecord} +import org.apache.linkis.storage.domain.Dolphin + +class TextResultSerializer extends ResultSerializer { + + override def metaDataToBytes(metaData: MetaData): Array[Byte] = { + if (metaData == null) { + lineToBytes(null) + } else { + val textMetaData = metaData.asInstanceOf[LineMetaData] + lineToBytes(textMetaData.getMetaData) + } + } + + override def recordToBytes(record: Record): Array[Byte] = { + val textRecord = record.asInstanceOf[LineRecord] + lineToBytes(textRecord.getLine) + } + + def lineToBytes(value: String): Array[Byte] = { + val bytes = if (value == null) Dolphin.NULL_BYTES else Dolphin.getBytes(value) + Dolphin.getIntBytes(bytes.length) ++ bytes + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSet.scala new file mode 100644 index 0000000000..00beb315c5 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/txt/TextResultSet.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset.txt + +import org.apache.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer} +import org.apache.linkis.storage.{LineMetaData, LineRecord} +import org.apache.linkis.storage.resultset.{ResultSetFactory, StorageResultSet} + +class TextResultSet extends StorageResultSet[LineMetaData, LineRecord] with Serializable { + + override def resultSetType(): String = ResultSetFactory.TEXT_TYPE + + override def createResultSetSerializer(): ResultSerializer = new TextResultSerializer + + override def createResultSetDeserializer(): ResultDeserializer[LineMetaData, LineRecord] = + new TextResultDeserializer + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsReader.scala new file mode 100644 index 0000000000..d89074b978 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsReader.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script + +import org.apache.linkis.common.io.{FsPath, FsReader} +import org.apache.linkis.storage.script.reader.StorageScriptFsReader + +import java.io.InputStream + +abstract class ScriptFsReader extends FsReader { + + val path: FsPath + val charset: String + +} + +object ScriptFsReader { + + def getScriptFsReader(path: FsPath, charset: String, inputStream: InputStream): ScriptFsReader = + new StorageScriptFsReader(path, charset, inputStream) + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala new file mode 100644 index 0000000000..dbcbfc126e --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/ScriptFsWriter.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script + +import org.apache.linkis.common.io.{FsPath, FsWriter, MetaData} +import org.apache.linkis.storage.LineRecord +import org.apache.linkis.storage.script.compaction.{ + PYScriptCompaction, + QLScriptCompaction, + ScalaScriptCompaction, + ShellScriptCompaction +} +import org.apache.linkis.storage.script.parser.{ + PYScriptParser, + QLScriptParser, + ScalaScriptParser, + ShellScriptParser +} +import org.apache.linkis.storage.script.writer.StorageScriptFsWriter + +import java.io.{InputStream, OutputStream} + +abstract class ScriptFsWriter extends FsWriter { + + val path: FsPath + val charset: String + + def getInputStream(): InputStream + +} + +object ScriptFsWriter { + + def getScriptFsWriter( + path: FsPath, + charset: String, + outputStream: OutputStream = null + ): ScriptFsWriter = + new StorageScriptFsWriter(path, charset, outputStream) + +} + +object ParserFactory { + + def listParsers(): Array[Parser] = + Array(PYScriptParser(), QLScriptParser(), ScalaScriptParser(), ShellScriptParser()) + +} + +object Compaction { + + def listCompactions(): Array[Compaction] = Array( + PYScriptCompaction(), + QLScriptCompaction(), + ScalaScriptCompaction(), + ShellScriptCompaction() + ) + +} + +trait Parser { + def prefixConf: String + + def prefix: String + + def belongTo(suffix: String): Boolean + + def parse(line: String): Variable + + def getAnnotationSymbol(): String +} + +trait Compaction { + + def prefixConf: String + + def prefix: String + + def belongTo(suffix: String): Boolean + + def compact(variable: Variable): String + + def getAnnotationSymbol(): String +} + +class ScriptMetaData(var variables: Array[Variable]) extends MetaData { + override def cloneMeta(): MetaData = new ScriptMetaData(variables) + + def getMetaData: Array[Variable] = variables + + def setMetaData(variables: Array[Variable]): Unit = { + this.variables = variables + } + +} + +class ScriptRecord(line: String) extends LineRecord(line) + +// definition variable; specialConfiguration ;runConfiguration; startUpConfiguration; +case class Variable(sortParent: String, sort: String, key: String, value: String) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala new file mode 100644 index 0000000000..ac89d19ea2 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/VariableParser.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script + +import java.util + +import scala.collection.mutable.ArrayBuffer + +object VariableParser { + + val CONFIGURATION: String = "configuration" + val VARIABLE: String = "variable" + val RUNTIME: String = "runtime" + val STARTUP: String = "startup" + val SPECIAL: String = "special" + + def getVariables(params: util.Map[String, Object]): Array[Variable] = { + import scala.collection.JavaConverters._ + val variables = new ArrayBuffer[Variable] + params + .getOrDefault(VARIABLE, new util.HashMap[String, Object]) + .asInstanceOf[util.Map[String, Object]] + .asScala + .foreach(f => variables += Variable(VARIABLE, null, f._1, f._2.toString)) + params + .getOrDefault(CONFIGURATION, new util.HashMap[String, Object]) + .asInstanceOf[util.Map[String, Object]] + .asScala + .foreach { f => + f._2 + .asInstanceOf[util.Map[String, Object]] + .asScala + .filter(s => !isContextIDINFO(s._1)) + .foreach(p => + p._2 match { + case e: util.Map[String, Object] => + e.asScala + .filter(s => !isContextIDINFO(s._1)) + .foreach(s => variables += Variable(f._1, p._1, s._1, s._2.toString)) + case _ => + if (null == p._2) { + variables += Variable(CONFIGURATION, f._1, p._1, "") + } else { + variables += Variable(CONFIGURATION, f._1, p._1, p._2.toString) + } + } + ) + } + variables.toArray + } + + // TODO need delete + def isContextIDINFO(key: String): Boolean = { + "contextID".equalsIgnoreCase(key) || "nodeName".equalsIgnoreCase(key) + } + + def getMap(variables: Array[Variable]): util.Map[String, Object] = { + import scala.collection.JavaConverters._ + val vars = new util.HashMap[String, String] + val confs = new util.HashMap[String, Object] + variables.filter(_.sort == null).foreach(f => vars.put(f.key, f.value)) + variables.filter(_.sort != null).foreach { f => + f.sort match { + case STARTUP | RUNTIME | SPECIAL => + if (confs.get(f.sort) == null) { + confs.put(f.sort, createMap(f)) + } else { + confs.get(f.sort).asInstanceOf[util.HashMap[String, Object]].put(f.key, f.value) + } + case _ => + if (confs.get(f.sortParent) == null) { + confs.put(f.sortParent, new util.HashMap[String, Object]) + confs + .get(f.sortParent) + .asInstanceOf[util.HashMap[String, Object]] + .put(f.sort, createMap(f)) + } else { + val subMap = confs.get(f.sortParent).asInstanceOf[util.HashMap[String, Object]] + if (subMap.get(f.sort) == null) { + subMap.put(f.sort, createMap(f)) + } else { + subMap + .get(f.sort) + .asInstanceOf[util.HashMap[String, Object]] + .put(f.key, f.value) + } + } + } + } + val params = new util.HashMap[String, Object] + if (vars.size() > 0) params.asScala += VARIABLE -> vars + if (confs.size() > 0) params.asScala += CONFIGURATION -> confs + params + } + + private def createMap(variable: Variable): util.Map[String, Object] = { + val map = new util.HashMap[String, Object] + map.put(variable.key, variable.value) + map + } + +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineMetaData.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala similarity index 60% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineMetaData.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala index a6dab8b37d..e085d2aacc 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/LineMetaData.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.scala @@ -15,30 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.storage; +package org.apache.linkis.storage.script.compaction -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.storage.resultset.ResultMetaData; +import org.apache.linkis.storage.script.{Compaction, Variable} -public class LineMetaData implements ResultMetaData { +abstract class CommonScriptCompaction extends Compaction { - private String metaData = null; - - public LineMetaData() {} - - public LineMetaData(String metaData) { - this.metaData = metaData; - } - - public String getMetaData() { - return metaData; + override def compact(variable: Variable): String = { + variable.sortParent match { + case "variable" => prefix + " " + variable.key + "=" + variable.value + case _ => + prefixConf + " " + variable.sortParent + " " + variable.sort + " " + variable.key + "=" + variable.value + } } - public void setMetaData(String metaData) { - this.metaData = metaData; - } + override def getAnnotationSymbol(): String = prefix.split('@')(0) - public MetaData cloneMeta() { - return new LineMetaData(metaData); - } } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/PYScriptCompaction.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/PYScriptCompaction.scala new file mode 100644 index 0000000000..a8e72a2d8f --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/PYScriptCompaction.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.compaction + +import org.apache.linkis.common.utils.CodeAndRunTypeUtils + +class PYScriptCompaction private extends CommonScriptCompaction { + + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON + ) + + override def prefix: String = "#@set" + + override def prefixConf: String = "#conf@set" +} + +object PYScriptCompaction { + val pYScriptCompaction: PYScriptCompaction = new PYScriptCompaction + + def apply(): CommonScriptCompaction = pYScriptCompaction +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/QLScriptCompaction.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/QLScriptCompaction.scala new file mode 100644 index 0000000000..7e420f8913 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/QLScriptCompaction.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.compaction + +import org.apache.linkis.common.utils.CodeAndRunTypeUtils + +class QLScriptCompaction private extends CommonScriptCompaction { + + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL + ) + + override def prefix: String = "--@set" + + override def prefixConf: String = "--conf@set" +} + +object QLScriptCompaction { + val qLScriptCompaction: QLScriptCompaction = new QLScriptCompaction + def apply(): CommonScriptCompaction = qLScriptCompaction +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.scala new file mode 100644 index 0000000000..c75c5a32a5 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/ScalaScriptCompaction.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.compaction + +import org.apache.linkis.common.utils.CodeAndRunTypeUtils + +class ScalaScriptCompaction private extends CommonScriptCompaction { + override def prefix: String = "//@set" + + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA + ) + + override def prefixConf: String = "//conf@set" +} + +object ScalaScriptCompaction { + private val compaction: ScalaScriptCompaction = new ScalaScriptCompaction + + def apply(): CommonScriptCompaction = compaction +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelFsWriter.java b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.scala similarity index 56% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelFsWriter.java rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.scala index cd3969e048..7a05580038 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelFsWriter.java +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/compaction/ShellScriptCompaction.scala @@ -15,27 +15,25 @@ * limitations under the License. */ -package org.apache.linkis.storage.excel; +package org.apache.linkis.storage.script.compaction -import org.apache.linkis.common.io.FsWriter; +import org.apache.linkis.common.utils.CodeAndRunTypeUtils -import java.io.OutputStream; +class ShellScriptCompaction private extends CommonScriptCompaction { + override def prefixConf: String = "#conf@set" -public abstract class ExcelFsWriter extends FsWriter { - public abstract String getCharset(); + override def prefix: String = "#@set" - public abstract String getSheetName(); + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL + ) - public abstract String getDateFormat(); +} - public abstract boolean isAutoFormat(); +object ShellScriptCompaction { + val shellScriptCompaction: ShellScriptCompaction = new ShellScriptCompaction - public static ExcelFsWriter getExcelFsWriter( - String charset, - String sheetName, - String dateFormat, - OutputStream outputStream, - boolean autoFormat) { - return new StorageExcelWriter(charset, sheetName, dateFormat, outputStream, autoFormat); - } + def apply(): CommonScriptCompaction = shellScriptCompaction } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala new file mode 100644 index 0000000000..b23a521cd2 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/CommonScriptParser.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.parser + +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.INVALID_CUSTOM_PARAMETER +import org.apache.linkis.storage.exception.StorageErrorException +import org.apache.linkis.storage.script.{Parser, Variable, VariableParser} + +abstract class CommonScriptParser extends Parser { + + @scala.throws[StorageErrorException] + def parse(line: String): Variable = { + val variableReg = ("\\s*" + prefix + "\\s*(.+)\\s*" + "=" + "\\s*(.+)\\s*").r + line match { + case variableReg(key, value) => + Variable(VariableParser.VARIABLE, null, key.trim, value.trim) + case _ => + val split = line.split("=") + if (split.length != 2) { + throw new StorageErrorException( + INVALID_CUSTOM_PARAMETER.getErrorCode(), + INVALID_CUSTOM_PARAMETER.getErrorDesc + ) + } + val value = split(1).trim + val subSplit = split(0).split(" ") + if (subSplit.filter(_ != "").size != 4) { + throw new StorageErrorException( + INVALID_CUSTOM_PARAMETER.getErrorCode(), + INVALID_CUSTOM_PARAMETER.getErrorDesc + ) + } + if (!subSplit.filter(_ != "")(0).equals(prefixConf)) { + throw new StorageErrorException( + INVALID_CUSTOM_PARAMETER.getErrorCode(), + INVALID_CUSTOM_PARAMETER.getErrorDesc + ) + } + val sortParent = subSplit.filter(_ != "")(1).trim + val sort = subSplit.filter(_ != "")(2).trim + val key = subSplit.filter(_ != "")(3).trim + Variable(sortParent, sort, key, value) + } + } + + override def getAnnotationSymbol(): String = prefix.split('@')(0) + +} diff --git a/linkis-public-enhancements/linkis-pes-common/src/test/scala/org/apache/linkis/errorcode/common/LinkisErrorCodeTest.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/PYScriptParser.scala similarity index 59% rename from linkis-public-enhancements/linkis-pes-common/src/test/scala/org/apache/linkis/errorcode/common/LinkisErrorCodeTest.scala rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/PYScriptParser.scala index 6e82f0ebcc..027b632146 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/test/scala/org/apache/linkis/errorcode/common/LinkisErrorCodeTest.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/PYScriptParser.scala @@ -15,21 +15,25 @@ * limitations under the License. */ -package org.apache.linkis.errorcode.common +package org.apache.linkis.storage.script.parser -import org.junit.jupiter.api.{Assertions, DisplayName, Test} +import org.apache.linkis.common.utils.CodeAndRunTypeUtils -class LinkisErrorCodeTest { +class PYScriptParser private extends CommonScriptParser { + override def prefix: String = "#@set" - @Test - @DisplayName("linkisErrorCodeTest") - def linkisErrorCodeTest(): Unit = { - val errorCode = new LinkisErrorCode( - "11000", - "Failed to get datasource info from datasource server(从数据源服务器获取数据源信息失败)" + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON ) - Assertions.assertEquals("11000", errorCode.getErrorCode) - } + override def prefixConf: String = "#conf@set" +} + +object PYScriptParser { + val pYScriptParser: PYScriptParser = new PYScriptParser + + def apply(): CommonScriptParser = pYScriptParser } diff --git a/linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/response/ScriptFromBMLResponseTest.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/QLScriptParser.scala similarity index 59% rename from linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/response/ScriptFromBMLResponseTest.scala rename to linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/QLScriptParser.scala index c4e9e386cb..a089f9c367 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/response/ScriptFromBMLResponseTest.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/QLScriptParser.scala @@ -15,24 +15,24 @@ * limitations under the License. */ -package org.apache.linkis.filesystem.response +package org.apache.linkis.storage.script.parser -import org.junit.jupiter.api.{Assertions, DisplayName, Test} +import org.apache.linkis.common.utils.CodeAndRunTypeUtils -class ScriptFromBMLResponseTest { +class QLScriptParser private extends CommonScriptParser { + override def prefix: String = "--@set" - @Test - @DisplayName("commonConst") - def commonConst(): Unit = { - val scriptContent = "show databases;" - val response = new ScriptFromBMLResponse( - scriptContent, - new java.util.HashMap[String, java.util.Map[String, Object]] + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL ) - Assertions.assertEquals(scriptContent, response.scriptContent) - Assertions.assertTrue(response.metadata.size() == 0) + override def prefixConf: String = "--conf@set" +} - } +object QLScriptParser { + val qLScriptParser: QLScriptParser = new QLScriptParser + def apply(): CommonScriptParser = qLScriptParser } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ScalaScriptParser.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ScalaScriptParser.scala new file mode 100644 index 0000000000..e6729326e8 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ScalaScriptParser.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.parser + +import org.apache.linkis.common.utils.CodeAndRunTypeUtils + +class ScalaScriptParser private extends CommonScriptParser { + // todo To be determined(待定) + override def prefix: String = "//@set" + + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA + ) + + override def prefixConf: String = "//conf@set" +} + +object ScalaScriptParser { + val otherScriptParser: ScalaScriptParser = new ScalaScriptParser + + def apply(): CommonScriptParser = otherScriptParser +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ShellScriptParser.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ShellScriptParser.scala new file mode 100644 index 0000000000..9b3e385ffe --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/parser/ShellScriptParser.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.parser + +import org.apache.linkis.common.utils.CodeAndRunTypeUtils + +class ShellScriptParser private extends CommonScriptParser { + override def prefix: String = "#@set" + + override def belongTo(suffix: String): Boolean = + CodeAndRunTypeUtils.getSuffixBelongToLanguageTypeOrNot( + suffix, + CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL + ) + + override def prefixConf: String = "#conf@set" +} + +object ShellScriptParser { + val shellScriptParser: ShellScriptParser = new ShellScriptParser + + def apply(): CommonScriptParser = shellScriptParser + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala new file mode 100644 index 0000000000..7b7b85ceb9 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.reader + +import org.apache.linkis.common.io.{FsPath, MetaData, Record} +import org.apache.linkis.storage.script._ +import org.apache.linkis.storage.utils.StorageUtils + +import org.apache.commons.io.IOUtils + +import java.io._ + +import scala.collection.mutable.ArrayBuffer + +class StorageScriptFsReader(val path: FsPath, val charset: String, val inputStream: InputStream) + extends ScriptFsReader { + + private var inputStreamReader: InputStreamReader = _ + private var bufferedReader: BufferedReader = _ + + private var metadata: ScriptMetaData = _ + + private var variables: ArrayBuffer[Variable] = new ArrayBuffer[Variable]() + private var lineText: String = _ + + @scala.throws[IOException] + override def getRecord: Record = { + + if (metadata == null) throw new IOException("Must read metadata first(必须先读取metadata)") + val record = new ScriptRecord(lineText) + lineText = bufferedReader.readLine() + record + } + + @scala.throws[IOException] + override def getMetaData: MetaData = { + if (metadata == null) init() + val parser = getScriptParser() + lineText = bufferedReader.readLine() + while (hasNext && parser != null && isMetadata(lineText, parser.prefix, parser.prefixConf)) { + variables += parser.parse(lineText) + lineText = bufferedReader.readLine() + } + metadata = new ScriptMetaData(variables.toArray) + metadata + } + + def init(): Unit = { + inputStreamReader = new InputStreamReader(inputStream) + bufferedReader = new BufferedReader(inputStreamReader) + } + + @scala.throws[IOException] + override def skip(recordNum: Int): Int = { + if (recordNum < 0) return -1 + if (metadata == null) getMetaData + try bufferedReader.skip(recordNum).toInt + catch { case t: Throwable => recordNum } + } + + @scala.throws[IOException] + override def getPosition: Long = -1L + + @scala.throws[IOException] + override def hasNext: Boolean = lineText != null + + @scala.throws[IOException] + override def available: Long = if (inputStream != null) inputStream.available() else 0L + + override def close(): Unit = { + IOUtils.closeQuietly(bufferedReader) + IOUtils.closeQuietly(inputStreamReader) + IOUtils.closeQuietly(inputStream) + } + + /** + * Determine if the read line is metadata(判断读的行是否是metadata) + * + * @param line + * @return + */ + def isMetadata(line: String, prefix: String, prefixConf: String): Boolean = { + val regex = ("\\s*" + prefix + "\\s*(.+)\\s*" + "=" + "\\s*(.+)\\s*").r + line match { + case regex(_, _) => true + case _ => + val split: Array[String] = line.split("=") + if (split.size != 2) return false + if (split(0).split(" ").filter(_ != "").size != 4) return false + if (!split(0).split(" ").filter(_ != "")(0).equals(prefixConf)) return false + true + } + } + + /** + * get the script parser according to the path(根据文件路径 获取对应的script parser ) + * @return + * Scripts Parser + */ + + def getScriptParser(): Parser = { + val parsers = + ParserFactory.listParsers().filter(p => p.belongTo(StorageUtils.pathToSuffix(path.getPath))) + if (parsers.length > 0) { + parsers(0) + } else { + null + } + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala new file mode 100644 index 0000000000..cdb9186da4 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.script.writer + +import org.apache.linkis.common.io.{FsPath, MetaData, Record} +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.LineRecord +import org.apache.linkis.storage.script.{Compaction, ScriptFsWriter, ScriptMetaData} +import org.apache.linkis.storage.utils.{StorageConfiguration, StorageUtils} + +import org.apache.commons.io.IOUtils +import org.apache.hadoop.hdfs.client.HdfsDataOutputStream + +import java.io.{ByteArrayInputStream, InputStream, IOException, OutputStream} +import java.util + +class StorageScriptFsWriter( + val path: FsPath, + val charset: String, + outputStream: OutputStream = null +) extends ScriptFsWriter + with Logging { + + private val stringBuilder = new StringBuilder + + @scala.throws[IOException] + override def addMetaData(metaData: MetaData): Unit = { + + val metadataLine = new util.ArrayList[String]() + val compaction = getScriptCompaction() + if (compaction != null) { + + metaData + .asInstanceOf[ScriptMetaData] + .getMetaData + .map(compaction.compact) + .foreach(metadataLine.add) + // add annotition symbol + if (metadataLine.size() > 0) { + metadataLine.add(compaction.getAnnotationSymbol()) + } + if (outputStream != null) { + IOUtils.writeLines(metadataLine, "\n", outputStream, charset) + } else { + import scala.collection.JavaConverters._ + metadataLine.asScala.foreach(m => stringBuilder.append(s"$m\n")) + } + } + + } + + @scala.throws[IOException] + override def addRecord(record: Record): Unit = { + // 转成LineRecord而不是TableRecord是为了兼容非Table类型的结果集写到本类中 + val scriptRecord = record.asInstanceOf[LineRecord] + if (outputStream != null) { + IOUtils.write(scriptRecord.getLine, outputStream, charset) + } else { + stringBuilder.append(scriptRecord.getLine) + } + } + + override def close(): Unit = { + if (outputStream != null) { + IOUtils.closeQuietly(outputStream) + } + } + + override def flush(): Unit = if (outputStream != null) { + Utils.tryAndWarnMsg[Unit] { + outputStream match { + case hdfs: HdfsDataOutputStream => + hdfs.hflush() + case _ => + outputStream.flush() + } + }(s"Error encounters when flush script ") + } + + def getInputStream(): InputStream = { + new ByteArrayInputStream( + stringBuilder.toString().getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue) + ) + } + + /** + * get the script compaction according to the path(根据文件路径 获取对应的script Compaction ) + * @return + * Scripts Compaction + */ + + def getScriptCompaction(): Compaction = { + + val compactions = Compaction + .listCompactions() + .filter(p => p.belongTo(StorageUtils.pathToSuffix(path.getPath))) + + if (compactions.length > 0) { + compactions(0) + } else { + null + } + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala new file mode 100644 index 0000000000..e0fa5b7f40 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.source + +import org.apache.linkis.common.io.{FsWriter, MetaData, Record} + +import org.apache.commons.io.IOUtils +import org.apache.commons.math3.util.Pair + +import java.util +import java.util.Arrays + +import scala.collection.JavaConverters._ + +abstract class AbstractFileSource(var fileSplits: Array[FileSplit]) extends FileSource { + + override def shuffle(s: Record => Record): FileSource = { + fileSplits.foreach(_.shuffler = s) + this + } + + override def page(page: Int, pageSize: Int): FileSource = { + fileSplits.foreach(_.page(page, pageSize)) + this + } + + override def addParams(params: util.Map[String, String]): FileSource = { + fileSplits.foreach(_.addParams(params)) + this + } + + override def addParams(key: String, value: String): FileSource = { + fileSplits.foreach(_.addParams(key, value)) + this + } + + override def getFileSplits: Array[FileSplit] = this.fileSplits + + override def getParams: util.Map[String, String] = + fileSplits.map(_.params.asScala).foldLeft(Map[String, String]())(_ ++ _).asJava + + override def write[K <: MetaData, V <: Record](fsWriter: FsWriter[K, V]): Unit = + fileSplits.foreach(_.write(fsWriter)) + + override def close(): Unit = this.fileSplits.foreach(IOUtils.closeQuietly) + + override def collect(): Array[Pair[Object, util.ArrayList[Array[String]]]] = + fileSplits.map(_.collect()) + + override def getTotalLine: Int = this.fileSplits.map(_.totalLine).sum + + override def getTypes: Array[String] = this.fileSplits.map(_.`type`) + + override def getFileInfo(needToCountRowNumber: Int = 5000): Array[Pair[Int, Int]] = + fileSplits.map(_.getFileInfo(needToCountRowNumber)) + + override def limitBytes(limitBytes: Long): FileSource = { + fileSplits.foreach((fileSplit: FileSplit) => fileSplit.setLimitBytes(limitBytes)) + this + } + + override def limitColumnLength(limitColumnLength: Int): FileSource = { + fileSplits.foreach((fileSplit: FileSplit) => fileSplit.setLimitColumnLength(limitColumnLength)) + this + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala new file mode 100644 index 0000000000..4b0b593db6 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.source + +import org.apache.linkis.common.io._ +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.storage.conf.LinkisStorageConf +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE +import org.apache.linkis.storage.exception.StorageErrorException +import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader} +import org.apache.linkis.storage.script.ScriptFsReader +import org.apache.linkis.storage.utils.StorageConfiguration + +import org.apache.commons.math3.util.Pair + +import java.io.{Closeable, InputStream} +import java.util + +trait FileSource extends Closeable { + + def shuffle(s: Record => Record): FileSource + + def page(page: Int, pageSize: Int): FileSource + + def collect(): Array[Pair[Object, util.ArrayList[Array[String]]]] + + def getFileInfo(needToCountRowNumber: Int = 5000): Array[Pair[Int, Int]] + + def write[K <: MetaData, V <: Record](fsWriter: FsWriter[K, V]): Unit + + def addParams(params: util.Map[String, String]): FileSource + + def addParams(key: String, value: String): FileSource + + def getParams: util.Map[String, String] + + def getTotalLine: Int + + def limitBytes(limitBytes: Long): FileSource + + def limitColumnLength(limitColumnLength: Int): FileSource + + def getTypes: Array[String] + + def getFileSplits: Array[FileSplit] +} + +object FileSource extends Logging { + + private val fileType = LinkisStorageConf.getFileTypeArr + private val suffixPredicate = (path: String, suffix: String) => path.endsWith(s".$suffix") + + def isResultSet(path: String): Boolean = { + suffixPredicate(path, fileType.head) + } + + def isResultSet(fsPath: FsPath): Boolean = { + isResultSet(fsPath.getPath) + } + + /** + * 目前只支持table多结果集 + * + * @param fsPaths + * @param fs + * @return + */ + def create(fsPaths: Array[FsPath], fs: Fs): FileSource = { + // 非table结果集的过滤掉 + val fileSplits = fsPaths.map(createResultSetFileSplit(_, fs)).filter(isTableResultSet) + new ResultsetFileSource(fileSplits) + } + + private def isTableResultSet(fileSplit: FileSplit): Boolean = + fileSplit.`type`.equals(ResultSetFactory.TABLE_TYPE) + + def isTableResultSet(fileSource: FileSource): Boolean = { + // 分片中全部为table结果集才返回true + fileSource.getFileSplits.forall(isTableResultSet) + } + + def create(fsPath: FsPath, fs: Fs): FileSource = { + if (!canRead(fsPath.getPath)) { + throw new StorageErrorException( + UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode, + UNSUPPORTED_OPEN_FILE_TYPE.getErrorDesc + ) + } + if (isResultSet(fsPath)) { + new ResultsetFileSource(Array(createResultSetFileSplit(fsPath, fs))) + } else { + new TextFileSource(Array(createTextFileSplit(fsPath, fs))) + } + } + + def create(fsPath: FsPath, is: InputStream): FileSource = { + if (!canRead(fsPath.getPath)) { + throw new StorageErrorException( + UNSUPPORTED_OPEN_FILE_TYPE.getErrorCode, + UNSUPPORTED_OPEN_FILE_TYPE.getErrorDesc + ) + } + if (isResultSet(fsPath)) { + new ResultsetFileSource(Array(createResultSetFileSplit(fsPath, is))) + } else { + new TextFileSource(Array(createTextFileSplit(fsPath, is))) + } + } + + private def createResultSetFileSplit(fsPath: FsPath, is: InputStream): FileSplit = { + val resultset = ResultSetFactory.getInstance.getResultSetByPath(fsPath) + val resultsetReader = ResultSetReader.getResultSetReader(resultset, is) + new FileSplit(resultsetReader, resultset.resultSetType()) + } + + private def createResultSetFileSplit(fsPath: FsPath, fs: Fs): FileSplit = { + logger.info(s"try create result set file split with path:${fsPath.getPath}") + val resultset = ResultSetFactory.getInstance.getResultSetByPath(fsPath, fs) + val resultsetReader = ResultSetReader.getResultSetReader(resultset, fs.read(fsPath)) + new FileSplit(resultsetReader, resultset.resultSetType()) + } + + private def createTextFileSplit(fsPath: FsPath, is: InputStream): FileSplit = { + val scriptFsReader = ScriptFsReader.getScriptFsReader( + fsPath, + StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue, + is + ) + new FileSplit(scriptFsReader) + } + + private def createTextFileSplit(fsPath: FsPath, fs: Fs): FileSplit = { + val scriptFsReader = ScriptFsReader.getScriptFsReader( + fsPath, + StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue, + fs.read(fsPath) + ) + new FileSplit(scriptFsReader) + } + + private def canRead(path: String): Boolean = { + fileType.exists(suffixPredicate(path, _)) + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala new file mode 100644 index 0000000000..f4163e8263 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.source + +import org.apache.linkis.common.io.{FsReader, FsWriter, MetaData, Record} +import org.apache.linkis.storage.{LineMetaData, LineRecord} +import org.apache.linkis.storage.domain.{Column, DataType} +import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} +import org.apache.linkis.storage.script.{ScriptMetaData, VariableParser} +import org.apache.linkis.storage.script.reader.StorageScriptFsReader + +import org.apache.commons.io.IOUtils +import org.apache.commons.math3.util.Pair + +import java.io.Closeable +import java.util + +import scala.collection.JavaConverters._ + +class FileSplit( + var fsReader: FsReader[_ <: MetaData, _ <: Record], + var `type`: String = "script/text" +) extends Closeable { + + var start: Int = 0 + + var end: Int = -1 + + var count: Int = 0 + + var totalLine = 0 + + var shuffler: Record => Record = r => r + + var pageTrigger: Boolean = false + + var params: util.Map[String, String] = new util.HashMap[String, String] + + private var limitBytes = 0L + private var limitColumnLength = 0 + + def page(page: Int, pageSize: Int): Unit = { + if (!pageTrigger) { + start = (page - 1) * pageSize + end = pageSize * page - 1 + pageTrigger = true + } + } + + def addParams(params: util.Map[String, String]): Unit = { + this.params.putAll(params) + } + + def addParams(key: String, value: String): Unit = { + this.params.put(key, value) + } + + def `while`[M](m: MetaData => M, r: Record => Unit): M = { + val metaData = fsReader.getMetaData + val t = m(metaData) + if (pageTrigger) { + fsReader.skip(start) + } + count = start + var hasRemovedFlag = false + while (fsReader.hasNext && ifContinueRead) { + val record = fsReader.getRecord + var needRemoveFlag = false + if (hasRemovedFlag == false && fsReader.isInstanceOf[StorageScriptFsReader]) { + val parser = fsReader.asInstanceOf[StorageScriptFsReader].getScriptParser() + val meta = metaData.asInstanceOf[ScriptMetaData].getMetaData + if ( + meta != null && meta.length > 0 + && parser != null && parser.getAnnotationSymbol().equals(record.toString) + ) { + needRemoveFlag = true + hasRemovedFlag = true + } + } + if (needRemoveFlag == false) { + r(shuffler(record)) + totalLine += 1 + count += 1 + } + } + t + } + + /** + * Get the colNumber and rowNumber of the row to be counted + * @param needToCountRowNumber + * @return + * colNumber, rowNumber + */ + def getFileInfo(needToCountRowNumber: Int = 5000): Pair[Int, Int] = { + val metaData = fsReader.getMetaData + val colNumber = metaData match { + case tableMetaData: TableMetaData => tableMetaData.columns.length + case _ => 1 + } + val rowNumber = if (needToCountRowNumber == -1) { + fsReader.skip(Int.MaxValue) + } else { + fsReader.skip(needToCountRowNumber) + } + new Pair(colNumber, rowNumber) + } + + def write[K <: MetaData, V <: Record](fsWriter: FsWriter[K, V]): Unit = { + `while`(fsWriter.addMetaData, fsWriter.addRecord) + } + + def collect(): Pair[Object, util.ArrayList[Array[String]]] = { + val record = new util.ArrayList[Array[String]] + var overFlag = false + var tmpBytes = 0L + + val metaData = `while`( + collectMetaData, + r => { + if (limitBytes > 0 && !overFlag) { + val resArr = collectRecord(r) + resArr.foreach(res => tmpBytes = tmpBytes + res.getBytes.length) + if (tmpBytes > limitBytes) { + overFlag = true + } + record.add(resArr) + } else { + record.add(collectRecord(r)) + } + } + ) + new Pair(metaData, record) + } + + def collectRecord(record: Record): Array[String] = { + record match { + case t: TableRecord => + if (limitColumnLength > 0) { + t.row.map { col => + val str = DataType.valueToString(col) + if (str.length > limitColumnLength) { + str.substring(0, limitColumnLength) + } else { + str + } + } + } else { + t.row.map(DataType.valueToString) + } + case l: LineRecord => Array(l.getLine) + } + } + + def collectMetaData(metaData: MetaData): Object = { + // script/text ,tableResultset,lineResultSet + metaData match { + case s: ScriptMetaData => VariableParser.getMap(s.getMetaData) + case l: LineMetaData => l.getMetaData + case t: TableMetaData => t.columns.map(ColumnToMap) + } + } + + private def ColumnToMap(column: Column): java.util.Map[String, String] = { + Map[String, String]( + "columnName" -> column.columnName, + "comment" -> column.comment, + "dataType" -> column.dataType.typeName + ) + }.asJava + + // 如果不分页,则一直读,如果分页,则 count需要小于count + def ifContinueRead: Boolean = !pageTrigger || count <= end + + def ifStartRead: Boolean = !pageTrigger || count >= start + + def setLimitBytes(limitBytes: Long): Unit = { + this.limitBytes = limitBytes + } + + def setLimitColumnLength(limitColumnLength: Int): Unit = { + this.limitColumnLength = limitColumnLength + } + + override def close(): Unit = IOUtils.closeQuietly(fsReader) + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala new file mode 100644 index 0000000000..adbb596aa2 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.source + +import org.apache.linkis.storage.domain.Dolphin +import org.apache.linkis.storage.resultset.table.TableRecord +import org.apache.linkis.storage.utils.StorageUtils + +class ResultsetFileSource(fileSplits: Array[FileSplit]) extends AbstractFileSource(fileSplits) { + + shuffle({ + case t: TableRecord => + new TableRecord(t.row.map { rvalue => + { + rvalue match { + case null | "NULL" => + val nullValue = getParams.getOrDefault("nullValue", "NULL") + if (nullValue.equals(Dolphin.LINKIS_NULL)) { + rvalue + } else { + nullValue + } + case "" => + val nullValue = getParams.getOrDefault("nullValue", "") + if (nullValue.equals(Dolphin.LINKIS_NULL)) { + "" + } else { + nullValue + } + case value: Double => StorageUtils.doubleToString(value) + case _ => rvalue + } + } + }) + case record => record + }) + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala new file mode 100644 index 0000000000..08ce8f51c7 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/TextFileSource.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.source + +import org.apache.linkis.storage.LineRecord +import org.apache.linkis.storage.script.ScriptRecord + +import org.apache.commons.math3.util.Pair + +import java.util + +import scala.collection.JavaConverters._ + +class TextFileSource(fileSplits: Array[FileSplit]) extends AbstractFileSource(fileSplits) { + + shuffle({ + case s: ScriptRecord if "".equals(s.getLine) => new LineRecord("\n") + case record => record + }) + + override def collect(): Array[Pair[Object, util.ArrayList[Array[String]]]] = { + val collects: Array[Pair[Object, util.ArrayList[Array[String]]]] = super.collect() + if (!getParams.getOrDefault("ifMerge", "true").toBoolean) return collects + val snds: Array[util.ArrayList[Array[String]]] = collects.map(_.getSecond) + snds.foreach { snd => + val str = new StringBuilder + snd.asScala.foreach { + case Array("\n") => str.append("\n") + case Array(y) => str.append(y).append("\n") + } + snd.clear() + snd.add(Array(str.toString())) + } + collects + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala new file mode 100644 index 0000000000..9c344fa802 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.utils + +import org.apache.linkis.common.io.FsPath +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.fs.impl.LocalFileSystem + +import java.io.IOException +import java.util + +object FileSystemUtils extends Logging { + + def copyFile(filePath: FsPath, origin: FsPath, user: String): Unit = { + val fileSystem = FSFactory.getFsByProxyUser(filePath, user).asInstanceOf[FileSystem] + Utils.tryFinally { + fileSystem.init(null) + if (!fileSystem.exists(filePath)) { + if (!fileSystem.exists(filePath.getParent)) { + fileSystem.mkdirs(filePath.getParent) + } + fileSystem.createNewFile(filePath) + } + fileSystem.copyFile(origin, filePath) + }(Utils.tryQuietly(fileSystem.close())) + } + + /** + * Create a new file(创建新文件) + * + * @param filePath + * @param createParentWhenNotExists + * Whether to recursively create a directory(是否递归创建目录) + */ + def createNewFile(filePath: FsPath, createParentWhenNotExists: Boolean): Unit = { + createNewFile(filePath, StorageUtils.getJvmUser, createParentWhenNotExists) + } + + def createNewFile(filePath: FsPath, user: String, createParentWhenNotExists: Boolean): Unit = { + val fileSystem = FSFactory.getFsByProxyUser(filePath, user).asInstanceOf[FileSystem] + Utils.tryFinally { + fileSystem.init(null) + createNewFileWithFileSystem(fileSystem, filePath, user, createParentWhenNotExists) + }(Utils.tryQuietly(fileSystem.close())) + } + + @deprecated("please use createNewFileAndSetOwnerWithFileSystem") + def createNewFileWithFileSystem( + fileSystem: FileSystem, + filePath: FsPath, + user: String, + createParentWhenNotExists: Boolean + ): Unit = { + if (!fileSystem.exists(filePath)) { + if (!fileSystem.exists(filePath.getParent)) { + if (!createParentWhenNotExists) { + throw new IOException("parent dir " + filePath.getParent.getPath + " dose not exists.") + } + mkdirs(fileSystem, filePath.getParent, user) + } + fileSystem.createNewFile(filePath) + fileSystem match { + case l: LocalFileSystem => fileSystem.setOwner(filePath, user) + case _ => logger.info(s"doesn't need to call setOwner") + } + } + } + + /** + * create new file and set file owner by FileSystem + * @param fileSystem + * @param filePath + * @param user + * @param createParentWhenNotExists + */ + def createNewFileAndSetOwnerWithFileSystem( + fileSystem: FileSystem, + filePath: FsPath, + user: String, + createParentWhenNotExists: Boolean + ): Unit = { + if (!fileSystem.exists(filePath)) { + if (!fileSystem.exists(filePath.getParent)) { + if (!createParentWhenNotExists) { + throw new IOException("parent dir " + filePath.getParent.getPath + " dose not exists.") + } + mkdirsAndSetOwner(fileSystem, filePath.getParent, user) + } + fileSystem.createNewFile(filePath) + fileSystem.setOwner(filePath, user) + } + } + + /** + * Recursively create a directory(递归创建目录) + * @param fileSystem + * @param dest + * @param user + * @throws + * @return + */ + @throws[IOException] + @deprecated("please use mkdirsAndSetOwner") + def mkdirs(fileSystem: FileSystem, dest: FsPath, user: String): Boolean = { + var parentPath = dest.getParent + val dirsToMake = new util.Stack[FsPath]() + dirsToMake.push(dest) + while (!fileSystem.exists(parentPath)) { + dirsToMake.push(parentPath) + parentPath = parentPath.getParent + } + if (!fileSystem.canExecute(parentPath)) { + throw new IOException("You have not permission to access path " + dest.getPath) + } + while (!dirsToMake.empty()) { + val path = dirsToMake.pop() + fileSystem.mkdir(path) + fileSystem match { + case l: LocalFileSystem => fileSystem.setOwner(path, user) + case _ => logger.info(s"doesn't need to call setOwner") + } + } + true + } + + /** + * Recursively create a directory(递归创建目录) 默认添加 Owner 信息 + * @param fileSystem + * @param dest + * @param user + * @throws + * @return + */ + @throws[IOException] + def mkdirsAndSetOwner(fileSystem: FileSystem, dest: FsPath, user: String): Boolean = { + var parentPath = dest.getParent + val dirsToMake = new util.Stack[FsPath]() + dirsToMake.push(dest) + while (!fileSystem.exists(parentPath)) { + dirsToMake.push(parentPath) + parentPath = parentPath.getParent + } + if (!fileSystem.canExecute(parentPath)) { + throw new IOException("You have not permission to access path " + dest.getPath) + } + while (!dirsToMake.empty()) { + val path = dirsToMake.pop() + fileSystem.mkdir(path) + fileSystem.setOwner(path, user) + } + true + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala new file mode 100644 index 0000000000..e73991db15 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.utils + +import org.apache.linkis.common.conf.{ByteType, CommonVars} + +object StorageConfiguration { + + val PROXY_USER = CommonVars("wds.linkis.storage.proxy.user", "${UM}") + + val STORAGE_ROOT_USER = CommonVars("wds.linkis.storage.root.user", "hadoop") + + val HDFS_ROOT_USER = CommonVars("wds.linkis.storage.hdfs.root.user", "hadoop") + + val LOCAL_ROOT_USER = CommonVars("wds.linkis.storage.local.root.user", "root") + + val STORAGE_USER_GROUP = CommonVars("wds.linkis.storage.fileSystem.group", "bdap") + + val STORAGE_RS_FILE_TYPE = CommonVars("wds.linkis.storage.rs.file.type", "utf-8") + + val STORAGE_RS_FILE_SUFFIX = CommonVars("wds.linkis.storage.rs.file.suffix", ".dolphin") + + val LINKIS_STORAGE_FS_LABEL = CommonVars("linkis.storage.default.fs.label", "linkis-storage") + + val ResultTypes = List("%TEXT", "%TABLE", "%HTML", "%IMG", "%ANGULAR", "%SVG") + + val STORAGE_RESULT_SET_PACKAGE = + CommonVars("wds.linkis.storage.result.set.package", "org.apache.linkis.storage.resultset") + + val STORAGE_RESULT_SET_CLASSES = CommonVars( + "wds.linkis.storage.result.set.classes", + "txt.TextResultSet,table.TableResultSet,io.IOResultSet,html.HtmlResultSet,picture.PictureResultSet" + ) + + val STORAGE_BUILD_FS_CLASSES = CommonVars( + "wds.linkis.storage.build.fs.classes", + "org.apache.linkis.storage.factory.impl.BuildHDFSFileSystem,org.apache.linkis.storage.factory.impl.BuildLocalFileSystem" + ) + + val IS_SHARE_NODE = CommonVars("wds.linkis.storage.is.share.node", true) + + val ENABLE_IO_PROXY = CommonVars("wds.linkis.storage.enable.io.proxy", false) + + val IO_USER = CommonVars("wds.linkis.storage.io.user", "root") + val IO_FS_EXPIRE_TIME = CommonVars("wds.linkis.storage.io.fs.num", 1000 * 60 * 10) + + val IO_PROXY_READ_FETCH_SIZE = + CommonVars("wds.linkis.storage.io.read.fetch.size", new ByteType("100k")) + + val IO_PROXY_WRITE_CACHE_SIZE = + CommonVars("wds.linkis.storage.io.write.cache.size", new ByteType("64k")) + + val IO_DEFAULT_CREATOR = CommonVars("wds.linkis.storage.io.default.creator", "IDE") + val IO_FS_RE_INIT = CommonVars("wds.linkis.storage.io.fs.re.init", "re-init") + + val IO_INIT_RETRY_LIMIT = CommonVars("wds.linkis.storage.io.init.retry.limit", 10) + + val STORAGE_HDFS_GROUP = CommonVars("wds.linkis.storage.fileSystem.hdfs.group", "hadoop") + + val DOUBLE_FRACTION_LEN = CommonVars[Int]("wds.linkis.double.fraction.length", 30) + + val HDFS_PATH_PREFIX_CHECK_ON = + CommonVars[Boolean]("wds.linkis.storage.hdfs.prefix_check.enable", true) + + val HDFS_PATH_PREFIX_REMOVE = CommonVars[Boolean]("wds.linkis.storage.hdfs.prefxi.remove", true) + + val FS_CHECKSUM_DISBALE = + CommonVars[java.lang.Boolean]("linkis.fs.hdfs.impl.disable.checksum", false) + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala new file mode 100644 index 0000000000..7e5169fc55 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageHelper.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.utils + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader} +import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} + +/** + * 工具类,用于做storage jar包打出来做测试用 Tool class, which is used to print the storage jar package for testing + */ + +object StorageHelper extends Logging { + + def main(args: Array[String]): Unit = { + if (args.length < 2) logger.info("Usage method params eg:getTableResLines path") + val method = args(0) + val params = args.slice(1, args.length) + Thread.sleep(10000L) + + method match { + case "getTableResLines" => getTableResLines(params) + case "getTableRes" => getTableRes(params) + case "createNewFile" => createNewFile(params) + case _ => logger.info("There is no such method") + } + } + + /** + * Get the number of table result set file lines(获得表格结果集文件行数) + * + * @param args + */ + def getTableResLines(args: Array[String]): Unit = { + val resPath = StorageUtils.getFsPath(args(0)) + val resultSetFactory = ResultSetFactory.getInstance + val resultSet = resultSetFactory.getResultSetByType(ResultSetFactory.TABLE_TYPE) + val fs = FSFactory.getFs(resPath) + fs.init(null) + val reader = ResultSetReader.getResultSetReader(resultSet, fs.read(resPath)) + val rmetaData = reader.getMetaData + rmetaData.asInstanceOf[TableMetaData].columns.foreach(column => logger.info(column.toString)) + var num = 0 + Thread.sleep(10000L) + while (reader.hasNext) { + reader.getRecord + num = num + 1 + } + logger.info(num.toString) + reader.close() + } + + def getTableRes(args: Array[String]): Unit = { + val len = Integer.parseInt(args(1)) + val max = len + 10 + val resPath = StorageUtils.getFsPath(args(0)) + val resultSetFactory = ResultSetFactory.getInstance + val resultSet = resultSetFactory.getResultSetByType(ResultSetFactory.TABLE_TYPE) + val fs = FSFactory.getFs(resPath) + fs.init(null) + val reader = ResultSetReader.getResultSetReader(resultSet, fs.read(resPath)) + val rmetaData = reader.getMetaData + rmetaData.asInstanceOf[TableMetaData].columns.foreach(column => logger.info(column.toString)) + rmetaData + .asInstanceOf[TableMetaData] + .columns + .map(_.columnName + ",") + .foreach(column => logger.info(column)) + var num = 0 + while (reader.hasNext) { + num = num + 1 + if (num > max) return + if (num > len) { + val record = reader.getRecord + record.asInstanceOf[TableRecord].row.foreach { value => + logger.info(value.toString) + logger.info(",") + } + logger.info("\n") + } + } + } + + def createNewFile(args: Array[String]): Unit = { + val resPath = StorageUtils.getFsPath(args(0)) + val proxyUser = StorageUtils.getJvmUser + FileSystemUtils.createNewFile(resPath, proxyUser, true) + logger.info("success") + } + +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala new file mode 100644 index 0000000000..4b9368c049 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageUtils.scala @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.utils + +import org.apache.linkis.common.io.{Fs, FsPath} +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.hadoop.common.conf.HadoopConf +import org.apache.linkis.storage.{LineMetaData, LineRecord} +import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.CONFIGURATION_NOT_READ +import org.apache.linkis.storage.exception.StorageFatalException +import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader, ResultSetWriter} + +import org.apache.commons.lang3.StringUtils + +import java.io.{Closeable, File, InputStream, OutputStream} +import java.lang.reflect.Method +import java.text.NumberFormat + +import scala.collection.mutable + +object StorageUtils extends Logging { + + val HDFS = "hdfs" + val FILE = "file" + + val FILE_SCHEMA = "file://" + val HDFS_SCHEMA = "hdfs://" + + private val nf = NumberFormat.getInstance() + nf.setGroupingUsed(false) + nf.setMaximumFractionDigits(StorageConfiguration.DOUBLE_FRACTION_LEN.getValue) + + def doubleToString(value: Double): String = { + if (value.isNaN) { + "NaN" + } else { + nf.format(value) + } + } + + def loadClass[T](classStr: String, op: T => String): Map[String, T] = { + val _classes = classStr.split(",") + val classes = mutable.LinkedHashMap[String, T]() + for (clazz <- _classes) { + Utils.tryAndError { + val obj = Utils.getClassInstance[T](clazz.trim) + classes += op(obj) -> obj + } + } + classes.toMap + } + + /** + * Get the corresponding class by passing in the subclass and package name(通过传入子类和包名获得对应的class) + * @param classStr：Class + * name(类名) + * @param pge:Class + * package name(类的包名) + * @param op：Get + * key value(获取键值) + * @tparam T + * @return + */ + def loadClasses[T]( + classStr: String, + pge: String, + op: Class[T] => String + ): Map[String, Class[T]] = { + val _classes: Array[String] = + if (StringUtils.isEmpty(pge)) classStr.split(",") + else classStr.split(",").map { value: String => pge + "." + value } + val classes = mutable.LinkedHashMap[String, Class[T]]() + for (clazz <- _classes) { + Utils.tryAndError({ + val _class = + Thread.currentThread.getContextClassLoader.loadClass(clazz.trim).asInstanceOf[Class[T]] + classes += op(_class) -> _class + }) + } + classes.toMap + } + + /** + * Get the suffix of the file name(获得文件名的后缀) + * @param path + * @return + */ + def pathToSuffix(path: String): String = { + val fileName = new File(path).getName + if ((fileName != null) && (fileName.length > 0)) { + val dot: Int = fileName.lastIndexOf('.') + if ((dot > -1) && (dot < (fileName.length - 1))) return fileName.substring(dot + 1) + } + fileName + } + + /** + * Reflection calling method(反射调用方法) + * @param obj + * @param method + * @param args + * @return + */ + def invoke(obj: Any, method: Method, args: Array[AnyRef]): Any = { + method.invoke(obj, args) + } + + /** + * Serialized string is a result set of type Text(序列化字符串为Text类型的结果集) + * @param value + * @return + */ + def serializerStringToResult(value: String): String = { + val resultSet = ResultSetFactory.getInstance.getResultSetByType(ResultSetFactory.TEXT_TYPE) + val writer = ResultSetWriter.getResultSetWriter(resultSet, Long.MaxValue, null) + val metaData = new LineMetaData() + val record = new LineRecord(value) + writer.addMetaData(metaData) + writer.addRecord(record) + val res = writer.toString() + Utils.tryQuietly(writer.close()) + res + } + + /** + * The result set of serialized text is a string(序列化text的结果集为字符串) + * @param result + * @return + */ + def deserializerResultToString(result: String): String = { + val resultSet = ResultSetFactory.getInstance.getResultSetByType(ResultSetFactory.TEXT_TYPE) + val reader = ResultSetReader.getResultSetReader(resultSet, result) + reader.getMetaData + val sb = new StringBuilder + while (reader.hasNext) { + val record = reader.getRecord.asInstanceOf[LineRecord] + sb.append(record.getLine) + } + val value = sb.toString() + Utils.tryQuietly(reader.close()) + value + } + + def close(outputStream: OutputStream): Unit = { + close(outputStream, null, null) + } + + def close(inputStream: InputStream): Unit = { + close(null, inputStream, null) + } + + def close(fs: Fs): Unit = { + close(null, null, fs) + } + + def close(outputStream: OutputStream, inputStream: InputStream, fs: Fs): Unit = { + Utils.tryFinally(if (outputStream != null) outputStream.close())() + Utils.tryFinally(if (inputStream != null) inputStream.close())() + Utils.tryFinally(if (fs != null) fs.close())() + } + + def close(closeable: Closeable): Unit = { + Utils.tryFinally(if (closeable != null) closeable.close())() + } + + def getJvmUser: String = System.getProperty("user.name") + + def isHDFSNode: Boolean = { + val confPath = new File(HadoopConf.hadoopConfDir) + // TODO IO-client mode need return false + if (!confPath.exists() || confPath.isFile) { + throw new StorageFatalException( + CONFIGURATION_NOT_READ.getErrorCode, + CONFIGURATION_NOT_READ.getErrorDesc + ) + } else true + } + + /** + * Returns the FsPath by determining whether the path is a schema. By default, the FsPath of the + * file is returned. 通过判断path是否为schema来返回FsPath，默认返回file的FsPath + * @param path + * @return + */ + def getFsPath(path: String): FsPath = { + if (path.startsWith(FILE_SCHEMA) || path.startsWith(HDFS_SCHEMA)) new FsPath(path) + else { + new FsPath(FILE_SCHEMA + path) + } + } + + def readBytes(inputStream: InputStream, bytes: Array[Byte], len: Int): Int = { + var count = 0 + var readLen = 0 + // 当使用s3存储结果文件时时，com.amazonaws.services.s3.model.S3InputStream无法正确读取.dolphin文件。需要在循环条件添加: + // readLen >= 0 + // To resolve the issue when using S3 to store result files and + // com.amazonaws.services.s3.model.S3InputStream to read .dolphin files, you need to add the + // condition readLen >= 0 in the loop. + while (readLen < len && readLen >= 0) { + count = inputStream.read(bytes, readLen, len - readLen) + if (count == -1 && inputStream.available() < 1) return readLen + readLen += count + } + readLen + } + + def isIOProxy(): Boolean = { + StorageConfiguration.ENABLE_IO_PROXY.getValue + } + + def isHDFSPath(fsPath: FsPath): Boolean = { + HDFS.equals(fsPath.getFsType) + } + +} diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/LineMetaDataTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/LineMetaDataTest.java deleted file mode 100644 index 902e835ec2..0000000000 --- a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/LineMetaDataTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage; - -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -class LineMetaDataTest { - - @Test - public void testCloneMeta() { - LineMetaData origin = new LineMetaData("origin"); - LineMetaData copied = (LineMetaData) origin.cloneMeta(); - origin.setMetaData(origin.getMetaData().replace("o", "a")); - - System.out.println(origin.getMetaData()); - Assertions.assertThat(origin.getMetaData()).isEqualTo("arigin"); - System.out.println(copied.getMetaData()); - Assertions.assertThat(copied.getMetaData()).isEqualTo("origin"); - } -} diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java new file mode 100644 index 0000000000..62260e8f64 --- /dev/null +++ b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.excel; + +import org.apache.commons.io.input.BOMInputStream; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.StringUtils; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import org.junit.jupiter.api.*; + +/** ExcelStorageReader Tester */ +public class ExcelStorageReaderTest { + + @Autowired private ExcelStorageReader excelStorageReader; + + @BeforeEach + @DisplayName("Each unit test method is executed once before execution") + public void before() throws Exception {} + + @AfterEach + @DisplayName("Each unit test method is executed once before execution") + public void after() throws Exception {} + + private static InputStream createExcelAndGetInputStream(int type) throws IOException { + Workbook workbook = new HSSFWorkbook(); + if (type == 1) { + workbook = new XSSFWorkbook(); + } + // 创建一个新的工作簿 + try { // HSSFWorkbook 用于处理 .xls 格式 + + // 创建一个工作表 + Sheet sheet = workbook.createSheet("Sheet1"); + + // 创建一行并在第一行写入一些数据（示例） + Row row = sheet.createRow(0); + Cell cell = row.createCell(0); + Cell cell2 = row.createCell(1); + cell.setCellValue("Hello"); + cell2.setCellValue("Hello2"); + + // 创建一个工作表 + Sheet sheet1 = workbook.createSheet("Sheet2"); + + // 创建一行并在第一行写入一些数据（示例） + Row row1 = sheet1.createRow(0); + Cell cell1 = row1.createCell(0); + Cell cell22 = row1.createCell(1); + cell1.setCellValue("Work"); + cell22.setCellValue("Work1"); + + // 将工作簿写入 ByteArrayOutputStream + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + workbook.write(outputStream); + + // 从 ByteArrayOutputStream 中获取 InputStream + return new ByteArrayInputStream(outputStream.toByteArray()); + } + } catch (Exception e) { + + } + return null; + } + + public InputStream createCSVInputStream(List> data) { + String csvData = convertToCSV(data); + return new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); + } + + private String convertToCSV(List> data) { + StringBuilder csvData = new StringBuilder(); + + for (List row : data) { + for (String column : row) { + csvData.append(column).append(","); + } + csvData.deleteCharAt(csvData.length() - 1).append("\n"); + } + + return csvData.toString(); + } + + private Map getCsvInfo(InputStream in, boolean escapeQuotes, boolean hasHeader) + throws Exception { + HashMap csvMap = new LinkedHashMap<>(); + String[][] column = null; + // fix csv file with utf-8 with bom chart[] + BOMInputStream bomIn = new BOMInputStream(in, false); // don't include the BOM + BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, "utf-8")); + + String header = reader.readLine(); + if (StringUtils.isEmpty(header)) { + throw new RuntimeException("内容为空"); + } + String[] line = header.split(",", -1); + int colNum = line.length; + column = new String[2][colNum]; + if (hasHeader) { + for (int i = 0; i < colNum; i++) { + column[0][i] = line[i]; + if (escapeQuotes) { + try { + csvMap.put(column[0][i].substring(1, column[0][i].length() - 1), "string"); + } catch (StringIndexOutOfBoundsException e) { + throw new RuntimeException("处理标题引号异常"); + } + } else { + csvMap.put(column[0][i], "string"); + } + } + } else { + for (int i = 0; i < colNum; i++) { + csvMap.put("col_" + (i + 1), "string"); + } + } + csvMap.forEach((key, value) -> System.out.println(key + ": " + value)); + return csvMap; + } + + @Test + public void getXlsSheetInfo() throws Exception { + Map>> sheetsInfo = + XlsUtils.getSheetsInfo(createExcelAndGetInputStream(0), true); + Assertions.assertTrue(sheetsInfo.containsKey("Sheet2")); + List> sheet2 = sheetsInfo.get("Sheet2"); + String work1 = ""; + for (Map sheetMap : sheet2) { + if (sheetMap.containsKey("Work1")) { + work1 = sheetMap.get("Work1"); + } + } + Assertions.assertEquals("string", work1); + } + + @Test + public void getXlsxSheetInfo() throws Exception { + Map>> sheetsInfo = + XlsxUtils.getAllSheetInfo(createExcelAndGetInputStream(1), null, true); + Assertions.assertTrue(sheetsInfo.containsKey("Sheet2")); + List> sheet2 = sheetsInfo.get("Sheet2"); + String work1 = ""; + for (Map sheetMap : sheet2) { + if (sheetMap.containsKey("Work1")) { + work1 = sheetMap.get("Work1"); + } + } + Assertions.assertEquals("string", work1); + } + + @Test + public void getCsvSheetInfo() throws Exception { + List> data = new ArrayList<>(); + data.add(Arrays.asList("Name", "Age", "City")); + data.add(Arrays.asList("John Doe", "30", "New York")); + data.add(Arrays.asList("Jane Smith", "25", "San Francisco")); + + // 有标题 + InputStream inputStream = createCSVInputStream(data); + Map csvMap = getCsvInfo(inputStream, false, true); + Assertions.assertEquals("string", csvMap.get("Name")); + + // 无标题 + InputStream inputStream1 = createCSVInputStream(data); + Map csvMap1 = getCsvInfo(inputStream1, false, false); + Assertions.assertEquals("string", csvMap1.get("col_1")); + + List> data1 = new ArrayList<>(); + data1.add(Arrays.asList("'Name'", "'Age'", "'City'")); + + // 有标题有引号 + InputStream inputStream2 = createCSVInputStream(data1); + Map csvMap2 = getCsvInfo(inputStream2, true, true); + Assertions.assertEquals("string", csvMap2.get("Name")); + + // 无标题 + InputStream inputStream3 = createCSVInputStream(data1); + Map csvMap3 = getCsvInfo(inputStream3, false, false); + Assertions.assertEquals("string", csvMap3.get("col_1")); + } +} diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/resultset/StorageResultSetWriterFactoryTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/resultset/StorageResultSetWriterFactoryTest.java deleted file mode 100644 index e7e79a7058..0000000000 --- a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/resultset/StorageResultSetWriterFactoryTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.resultset; - -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSet; -import org.apache.linkis.storage.LineMetaData; -import org.apache.linkis.storage.LineRecord; - -import java.io.*; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class StorageResultSetWriterFactoryTest { - - @Test - void testResultSetWrite() throws IOException { - // storage write - ResultSet resultSetByType = - ResultSetFactory.getInstance().getResultSetByType(ResultSetFactory.TEXT_TYPE); - - org.apache.linkis.common.io.resultset.ResultSetWriter - writer = ResultSetWriterFactory.getResultSetWriter(resultSetByType, 100L, null); - - String value = "value"; - LineMetaData metaData = new LineMetaData(null); - LineRecord record = new LineRecord(value); - writer.addMetaData(metaData); - writer.addRecord(record); - writer.flush(); - writer.close(); - String res = writer.toString(); - writer.close(); - Assertions.assertEquals("dolphin00000000010000000004NULL0000000005value", res); - } -} diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java index 2e1a30aa27..60a593665c 100644 --- a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java +++ b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java @@ -25,6 +25,7 @@ import org.apache.linkis.storage.script.Variable; import org.apache.linkis.storage.script.VariableParser; import org.apache.linkis.storage.source.FileSource; +import org.apache.linkis.storage.source.FileSource$; import org.apache.commons.math3.util.Pair; @@ -37,6 +38,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -105,7 +107,7 @@ void TestSave() { Variable[] v = VariableParser.getVariables(params); List variableList = Arrays.stream(v) - .filter(var -> !StringUtils.isEmpty(var.getValue())) + .filter(var -> !StringUtils.isEmpty(var.value())) .collect(Collectors.toList()); try { @@ -136,8 +138,8 @@ void TestOpen() throws FileNotFoundException { InputStream inputStream = new FileInputStream(file); - FileSource fileSource = FileSource.create(new FsPath(fileName), inputStream); - Pair> collect = fileSource.collect()[0]; + FileSource fileSource = FileSource$.MODULE$.create(new FsPath(fileName), inputStream); + Pair> collect = fileSource.collect()[0]; String scriptRes = collect.getSecond().get(0)[0]; String metadataRes = new Gson().toJson(collect.getFirst()); diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/source/ResultsetFileSourceTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/source/ResultsetFileSourceTest.java deleted file mode 100644 index 1210c64e03..0000000000 --- a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/source/ResultsetFileSourceTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.storage.source; - -import org.apache.linkis.common.io.Fs; -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.storage.FSFactory; -import org.apache.linkis.storage.csv.CSVFsWriter; - -import java.io.IOException; -import java.io.OutputStream; - -import org.junit.jupiter.api.Test; - -class ResultsetFileSourceTest { - - @Test - public void testWriter() throws IOException { - String filePath = this.getClass().getResource("/result-read-test.dolphin").getFile().toString(); - FsPath sourceFsPath = new FsPath(filePath); - Fs sourceFs = FSFactory.getFs(sourceFsPath); - sourceFs.init(null); - - FsPath destFsPath = new FsPath(filePath + ".result"); - Fs destFs = FSFactory.getFs(destFsPath); - destFs.init(null); - OutputStream outputStream = destFs.write(destFsPath, true); - - CSVFsWriter cSVFsWriter = CSVFsWriter.getCSVFSWriter("UTF-8", ",", false, outputStream); - FileSource fileSource = FileSource.create(sourceFsPath, sourceFs); - fileSource.addParams("nullValue", "NULL").write(cSVFsWriter); - - cSVFsWriter.close(); - } -} diff --git a/linkis-commons/linkis-storage/src/test/resources/result-read-test.dolphin b/linkis-commons/linkis-storage/src/test/resources/result-read-test.dolphin deleted file mode 100644 index 0c48c045b0..0000000000 --- a/linkis-commons/linkis-storage/src/test/resources/result-read-test.dolphin +++ /dev/null @@ -1 +0,0 @@ -dolphin0000000002000000002900000000063,6,4,_c0bigintNULL000000001400000000022,55 \ No newline at end of file diff --git a/linkis-commons/linkis-storage/src/test/resources/storage-read-test.dolphin b/linkis-commons/linkis-storage/src/test/resources/storage-read-test.dolphin deleted file mode 100644 index b94e3d96fb..0000000000 --- a/linkis-commons/linkis-storage/src/test/resources/storage-read-test.dolphin +++ /dev/null @@ -1 +0,0 @@ -dolphin00000000010000000004NULL0000000011hello world \ No newline at end of file diff --git a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala index 3bfc35b17c..e0d3d7efb6 100644 --- a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala @@ -25,7 +25,7 @@ class DataTypeTest { @DisplayName("constTest") def constTest(): Unit = { - val nullvalue = DataType.NULL_VALUE + val nullvalue = Dolphin.NULL val lowcasenullvalue = DataType.LOWCASE_NULL_VALUE Assertions.assertEquals("NULL", nullvalue) @@ -60,4 +60,19 @@ class DataTypeTest { } + @Test + @DisplayName("toValueTest") + def toValueTest(): Unit = { + val dateType = DataType.toDataType("double") + val str = DataType.toValue(dateType, "NaN") + Assertions.assertNotNull(str) + } + + @Test + @DisplayName("decimalTest") + def decimalTest(): Unit = { + val dateType = DataType.toDataType("decimal(10, 8)") + Assertions.assertTrue(dateType.typeName.equals("decimal")) + } + } diff --git a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageConfigurationTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageConfigurationTest.scala index 6534b25c6f..f4082c65eb 100644 --- a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageConfigurationTest.scala +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageConfigurationTest.scala @@ -26,7 +26,7 @@ class StorageConfigurationTest { def constTest(): Unit = { val storagerootuser = StorageConfiguration.STORAGE_ROOT_USER.getValue - val hdfsrootuser = StorageConfiguration.HDFS_ROOT_USER.getValue + val hdfsrootuser = StorageConfiguration.HDFS_ROOT_USER().getValue val localrootuser = StorageConfiguration.LOCAL_ROOT_USER.getValue val storageusergroup = StorageConfiguration.STORAGE_USER_GROUP.getValue val storagersfiletype = StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue @@ -46,7 +46,6 @@ class StorageConfigurationTest { val doublefractionlen = StorageConfiguration.DOUBLE_FRACTION_LEN.getValue val hdfspathprefixcheckon = StorageConfiguration.HDFS_PATH_PREFIX_CHECK_ON.getValue val hdfspathprefixremove = StorageConfiguration.HDFS_PATH_PREFIX_REMOVE.getValue - val fscachedisable = StorageConfiguration.FS_CACHE_DISABLE.getValue val fschecksumdisbale = StorageConfiguration.FS_CHECKSUM_DISBALE.getValue Assertions.assertEquals("hadoop", storagerootuser) @@ -62,8 +61,7 @@ class StorageConfigurationTest { storageresultsetclasses ) Assertions.assertEquals( - "org.apache.linkis.storage.factory.impl.BuildHDFSFileSystem,org.apache.linkis.storage.factory.impl.BuildLocalFileSystem," + - "org.apache.linkis.storage.factory.impl.BuildOSSSystem,org.apache.linkis.storage.factory.impl.BuildS3FileSystem", + "org.apache.linkis.storage.factory.impl.BuildHDFSFileSystem,org.apache.linkis.storage.factory.impl.BuildLocalFileSystem", storagebuildfsclasses ) Assertions.assertTrue(issharenode) @@ -77,7 +75,6 @@ class StorageConfigurationTest { Assertions.assertTrue(30 == doublefractionlen) Assertions.assertTrue(hdfspathprefixcheckon) Assertions.assertTrue(hdfspathprefixremove) - Assertions.assertFalse(fscachedisable) Assertions.assertFalse(fschecksumdisbale) } diff --git a/linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientImplTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala similarity index 72% rename from linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientImplTest.scala rename to linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala index 83408ed7e8..6ae15782ee 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/test/scala/org/apache/linkis/filesystem/WorkspaceClientImplTest.scala +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala @@ -15,22 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.filesystem +package org.apache.linkis.storage.utils import org.junit.jupiter.api.{Assertions, DisplayName, Test} -class WorkspaceClientImplTest { +class StorageUtilsTest { @Test - @DisplayName("createObject") - def createObject(): Unit = { + @DisplayName("doubleToStringTest") + def doubleToStringTest(): Unit = { + val str = StorageUtils.doubleToString(Double.NaN) + Assertions.assertEquals("NaN", str) - val user = "hadoop" - val token = "abc" - val gatewayAddress = "127.0.0.1:9001" - - val client = new WorkspaceClientImpl(user, token, gatewayAddress) - Assertions.assertNotNull(client) } } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java deleted file mode 100644 index 739c256af1..0000000000 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.governance.common.utils; - -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; - -import java.io.File; -import java.nio.file.Paths; - -public class ECPathUtils { - - public static String getECWOrkDirPathSuffix(String user, String ticketId, String engineType) { - String engineTypeRes = ""; - if (StringUtils.isNotBlank(engineType)) { - engineTypeRes = engineType; - } - File file = - Paths.get( - user, DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd"), engineTypeRes) - .toFile(); - return file.getPath() + File.separator + ticketId; - } - - public static String getECLogDirSuffix( - EngineTypeLabel engineTypeLabel, UserCreatorLabel userCreatorLabel, String ticketId) { - if (null == engineTypeLabel || null == userCreatorLabel) { - return ""; - } - String ecwOrkDirPathSuffix = - ECPathUtils.getECWOrkDirPathSuffix( - userCreatorLabel.getUser(), ticketId, engineTypeLabel.getEngineType()); - return ecwOrkDirPathSuffix + File.separator + "logs"; - } -} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala similarity index 81% rename from linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala rename to linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala index b8b156173b..b0c9dda666 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala @@ -18,7 +18,7 @@ package org.apache.linkis.governance.common.conf import org.apache.linkis.common.conf.{CommonVars, Configuration} -import org.apache.linkis.manager.label.conf.LabelCommonConfig +import org.apache.linkis.governance.common.constant.ec.ECConstants object GovernanceCommonConf { @@ -26,11 +26,9 @@ object GovernanceCommonConf { val WILDCARD_CONSTANT = "*" - val SPARK_ENGINE_VERSION = - CommonVars("wds.linkis.spark.engine.version", LabelCommonConfig.SPARK_ENGINE_VERSION.getValue) + val SPARK_ENGINE_VERSION = CommonVars("wds.linkis.spark.engine.version", "2.4.3") - val HIVE_ENGINE_VERSION = - CommonVars("wds.linkis.hive.engine.version", LabelCommonConfig.HIVE_ENGINE_VERSION.getValue) + val HIVE_ENGINE_VERSION = CommonVars("wds.linkis.hive.engine.version", "1.2.1") val PYTHON_ENGINE_VERSION = CommonVars("wds.linkis.python.engine.version", "python2") @@ -46,9 +44,7 @@ object GovernanceCommonConf { val ENGINE_APPLICATION_MANAGER_SPRING_NAME = CommonVars("wds.linkis.application.manager.name", "linkis-cg-linkismanager") - val ENGINE_CONN_PORT_RANGE = CommonVars("linkis.engineconn.port.range", "-") - - val ENGINE_CONN_DEBUG_PORT_RANGE = CommonVars("linkis.engineconn.debug.port.range", "-") + val ENGINE_CONN_PORT_RANGE = CommonVars("wds.linkis.engineconn.port.range", "-") val MANAGER_SERVICE_NAME = CommonVars( @@ -90,4 +86,13 @@ object GovernanceCommonConf { val EC_APP_MANAGE_MODE = CommonVars("linkis.ec.app.manage.mode", "attach") + /** + * DEFAULT_LOGPATH_PREFIX is the prefix that represents the default log storage path + * DEFAULT_LOGPATH_PREFIX 是表示默认的日志存储路径的前缀 和 结果集的前缀 + */ + val DEFAULT_LOGPATH_PREFIX = CommonVars[String]( + "wds.linkis.entrance.config.log.path", + CommonVars[String]("wds.linkis.filesystem.hdfs.root.path").getValue + ).getValue + } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala index 9f2879d571..fef9120853 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala @@ -17,11 +17,6 @@ package org.apache.linkis.governance.common.constant.job -/** - * @date - * 2021/3/17 - * @description - */ object JobRequestConstants { val JOB_ID = "jobId" diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala index ec7bb9e80a..4168308bd6 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/exception/GovernanceErrorException.scala @@ -17,15 +17,14 @@ package org.apache.linkis.governance.common.exception -import org.apache.linkis.common.exception.{ExceptionLevel, LinkisRuntimeException} +import org.apache.linkis.common.exception.ErrorException class GovernanceErrorException(errorCode: Int, errorMsg: String) - extends LinkisRuntimeException(errorCode, errorMsg) { + extends ErrorException(errorCode, errorMsg) { def this(errorCode: Int, errorMsg: String, cause: Throwable) = { this(errorCode, errorMsg) initCause(cause) } - override def getLevel: ExceptionLevel = ExceptionLevel.ERROR } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/RequestTask.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/RequestTask.scala index 17c01fcfc2..4d0b8952ca 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/RequestTask.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/RequestTask.scala @@ -18,7 +18,6 @@ package org.apache.linkis.governance.common.protocol.task import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.protocol.RetryableProtocol import org.apache.linkis.protocol.message.RequestProtocol import java.util @@ -92,7 +91,7 @@ trait TaskState extends RequestProtocol {} case class RequestTaskPause(execId: String) extends TaskState case class RequestTaskResume(execId: String) extends TaskState -case class RequestTaskKill(execId: String) extends TaskState with RetryableProtocol +case class RequestTaskKill(execId: String) extends TaskState /** * The status of requesting job execution, mainly used for:
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseEngineConnPid.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseEngineConnPid.scala index ef1355d580..971bdf247b 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseEngineConnPid.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseEngineConnPid.scala @@ -25,4 +25,5 @@ import org.apache.linkis.protocol.message.RequestProtocol * @param pid */ case class ResponseEngineConnPid(serviceInstance: ServiceInstance, pid: String, ticketId: String) - extends RequestProtocol + extends RetryableProtocol + with RequestProtocol diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseTaskExecute.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseTaskExecute.scala index a4a7837da0..f59761dc43 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseTaskExecute.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/task/ResponseTaskExecute.scala @@ -24,8 +24,6 @@ import org.apache.linkis.protocol.message.RequestProtocol import java.util -case class ResponseTaskExecute(execId: String) - case class ResponseTaskProgress( execId: String, progress: Float, @@ -41,31 +39,6 @@ case class EngineConcurrentInfo( failedTasks: Int ) -case class EngineOverloadInfo(maxMemory: Long, usedMemory: Long, systemCPUUsed: Float) - -case class ResponseEngineStatusChanged( - instance: String, - fromStatus: ExecutionNodeStatus, - toStatus: ExecutionNodeStatus, - overload: EngineOverloadInfo, - concurrent: EngineConcurrentInfo -) extends BroadcastProtocol - -case class ResponseEngineInfo( - createEntranceInstance: String, - creator: String, - user: String, - properties: util.Map[String, String] -) - -case class ResponseEngineStatus( - instance: String, - status: ExecutionNodeStatus, - overload: EngineOverloadInfo, - concurrent: EngineConcurrentInfo, - engineInfo: ResponseEngineInfo -) - case class ResponseTaskLog(execId: String, log: String) extends RequestProtocol case class ResponseTaskError(execId: String, errorMsg: String) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala new file mode 100644 index 0000000000..236046f3d4 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.utils + +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} + +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateFormatUtils + +import java.io.File +import java.nio.file.Paths + +object ECPathUtils { + + def getECWOrkDirPathSuffix( + user: String, + ticketId: String, + engineType: String, + timeStamp: Long = System.currentTimeMillis() + ): String = { + val suffix = if (StringUtils.isBlank(engineType)) { + Paths + .get(user, DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd")) + .toFile + .getPath + } else { + Paths + .get(user, DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd"), engineType) + .toFile + .getPath + } + suffix + File.separator + ticketId + } + + def getECLogDirSuffix( + engineTypeLabel: EngineTypeLabel, + userCreatorLabel: UserCreatorLabel, + ticketId: String + ): String = { + if (null == engineTypeLabel || null == userCreatorLabel) { + return "" + } + val suffix = ECPathUtils.getECWOrkDirPathSuffix( + userCreatorLabel.getUser, + ticketId, + engineTypeLabel.getEngineType + ) + suffix + File.separator + "logs" + } + +} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/EngineConnArguments.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/EngineConnArguments.scala index 684dd371a1..31d9b1e9b2 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/EngineConnArguments.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/EngineConnArguments.scala @@ -145,9 +145,13 @@ class DefaultEngineConnArgumentsParser extends EngineConnArgumentsParser { val options = ArrayBuffer[String]() def write(confMap: Map[String, String], optionType: String): Unit = confMap.foreach { case (key, value) => - if (StringUtils.isNotEmpty(key) && StringUtils.isNotEmpty(value)) { + var realValue = value + if (key.startsWith("label") && StringUtils.isEmpty(realValue)) { + realValue = "true" + } + if (StringUtils.isNotEmpty(key) && StringUtils.isNotEmpty(realValue)) { options += optionType - options += (key + "=" + value) + options += (key + "=" + realValue) } } write(engineConnArguments.getEngineConnConfMap, ENGINE_CONN_CONF) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala index ddcb17a3b2..ae83749ecb 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala @@ -24,11 +24,14 @@ import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.commons.lang3.StringUtils import java.io.File +import java.text.SimpleDateFormat import java.util -import java.util.{ArrayList, List} +import java.util.{ArrayList, Date, List} object GovernanceUtils extends Logging { + val LINKIS_DEFAULT_RES_CREATOR = "linkis_default" + def killProcess(pid: String, desc: String, isSudo: Boolean): Unit = { val subProcessKillScriptPath = Configuration.getLinkisHome() + "/sbin/kill-process-by-pid.sh" if ( @@ -121,4 +124,26 @@ object GovernanceUtils extends Logging { } } + /** + * get result path parentPath: resPrefix + dateStr + result + creator subPath: parentPath + + * executeUser + taskid + filename + * + * @param creator + * @return + */ + def getResultParentPath(creator: String): String = { + val resPrefix = GovernanceCommonConf.DEFAULT_LOGPATH_PREFIX + val resStb = new StringBuilder() + if (resStb.endsWith("/")) { + resStb.append(resPrefix) + } else { + resStb.append(resPrefix).append("/") + } + val dateFormat = new SimpleDateFormat("yyyy-MM-dd") + val date = new Date(System.currentTimeMillis) + val dateString = dateFormat.format(date) + resStb.append("result").append("/").append(dateString).append("/").append(creator) + resStb.toString() + } + } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/OnceExecutorContentUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/OnceExecutorContentUtils.scala index 2c426339b0..dd4b9bcffa 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/OnceExecutorContentUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/OnceExecutorContentUtils.scala @@ -58,14 +58,14 @@ object OnceExecutorContentUtils { def mapToContent(contentMap: util.Map[String, Object]): OnceExecutorContent = { val onceExecutorContent = new OnceExecutorContent - def getOrNull(key: String): util.Map[String, Object] = contentMap.get(key) match { + implicit def getOrNull(key: String): util.Map[String, Object] = contentMap.get(key) match { case map: util.Map[String, Object] => map case _ => null } - onceExecutorContent.setJobContent(getOrNull(TaskConstant.JOB_CONTENT)) - onceExecutorContent.setRuntimeMap(getOrNull(TaskConstant.PARAMS_CONFIGURATION_RUNTIME)) - onceExecutorContent.setSourceMap(getOrNull(TaskConstant.SOURCE)) - onceExecutorContent.setVariableMap(getOrNull(TaskConstant.PARAMS_VARIABLE)) + onceExecutorContent.setJobContent(TaskConstant.JOB_CONTENT) + onceExecutorContent.setRuntimeMap(TaskConstant.PARAMS_CONFIGURATION_RUNTIME) + onceExecutorContent.setSourceMap(TaskConstant.SOURCE) + onceExecutorContent.setVariableMap(TaskConstant.PARAMS_VARIABLE) onceExecutorContent } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala index 5b23d01fc3..cc79e24d4f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala @@ -42,7 +42,6 @@ import org.apache.commons.lang3.StringUtils import java.io.{File, InputStream, OutputStream} import scala.collection.JavaConverters._ -import scala.collection.mutable trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging { @@ -102,12 +101,7 @@ trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging { case HIVE_CONF_DIR => putIfExists(HIVE_CONF_DIR) case JAVA_HOME => putIfExists(JAVA_HOME) case RANDOM_PORT => - environment.put( - RANDOM_PORT.toString, - PortUtils - .findAvailPortByRange(GovernanceCommonConf.ENGINE_CONN_DEBUG_PORT_RANGE.getValue) - .toString - ) + environment.put(RANDOM_PORT.toString, PortUtils.findAvailPort().toString) case PREFER_IP_ADDRESS => environment.put(PREFER_IP_ADDRESS.toString, Configuration.PREFER_IP_ADDRESS.toString) case ENGINECONN_ENVKEYS => @@ -157,19 +151,15 @@ trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging { def getPid(): Option[String] = None protected def getCommandArgs: Array[String] = { - val recordMap: mutable.Map[String, String] = mutable.Map() - request.creationDesc.properties.asScala.foreach { case (k, v) => - if (k.contains(" ") || (v != null && v.contains(" "))) recordMap.put(k, v) - } - if (recordMap.size > 0) { - val keyAndValue = new StringBuilder - for (kv <- recordMap) { - keyAndValue.append(s"${kv._1}->${kv._2};") - } + if ( + request.creationDesc.properties.asScala.exists { case (k, v) => + k.contains(" ") || (v != null && v.contains(" ")) + } + ) { throw new ErrorException( 30000, - s"Startup parameters contain spaces! The key and value values of all its parameters are(启动参数中包含空格！其所有参数的key和value值分别为)：${keyAndValue.toString()}" - ) + "Startup parameters contain spaces!(启动参数中包含空格！)" + ) // TODO exception } val arguments = EngineConnArgumentsBuilder.newBuilder() engineConnPort = PortUtils diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala index 21dd6cd706..8552020493 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala @@ -44,7 +44,7 @@ object PortUtils extends Logging { val ports = portRange.split(separator, -1) if (!ports(0).isEmpty) start = ports(0).toInt if (!ports(1).isEmpty) end = ports(1).toInt - val availablePort = start to end find { port => + val availablePort = start until end find { port => try { new ServerSocket(port).close() true diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/assembly/distribution.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/assembly/distribution.xml index da4e71662b..f24c7e21a1 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/assembly/distribution.xml +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/assembly/distribution.xml @@ -244,9 +244,6 @@ org.slf4j:jul-to-slf4j:jar org.slf4j:slf4j-api:jar org.springframework.boot:spring-boot:jar - org.springframework.boot:spring-boot-actuator:jar - org.springframework.boot:spring-boot-actuator-autoconfigure:jar - org.springframework.boot:spring-boot-autoconfigure:jar org.springframework.boot:spring-boot-starter:jar org.springframework.boot:spring-boot-starter-actuator:jar org.springframework.boot:spring-boot-starter-aop:jar diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/restful/ECMRestfulApi.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/restful/ECMRestfulApi.java index d4fc0a49e1..2de878a24c 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/restful/ECMRestfulApi.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/restful/ECMRestfulApi.java @@ -152,7 +152,7 @@ public Message downloadEngineLog( outputStream.write(buffer, 0, bytesRead); } } catch (IOException e) { - logger.warn("Download EngineLog Failed Msg :", e); + logger.error("Download EngineLog Failed Msg :", e); response.reset(); response.setCharacterEncoding(Consts.UTF_8.toString()); response.setContentType("text/plain; charset=utf-8"); diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/scheduled/EcmClearTask.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/scheduled/EcmClearTask.java index 111ad896e0..409f0d7e98 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/scheduled/EcmClearTask.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/scheduled/EcmClearTask.java @@ -46,7 +46,7 @@ public void run() { cmdlist.add("sh"); cmdlist.add(shellPath + "linkis-ec-clear.sh"); try { - Utils.exec(cmdlist.toArray(new String[0]), 3000L); + Utils.exec(cmdlist.toArray(new String[0]), 1800000L); } catch (Exception e) { logger.warn("Shell linkis-ec-clear.sh execution failed, msg:" + e.getMessage()); } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java deleted file mode 100644 index 3d62df5c90..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.operator; - -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.ecm.server.conf.ECMConfiguration; -import org.apache.linkis.ecm.server.exception.ECMErrorException; -import org.apache.linkis.manager.common.operator.Operator; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.ReversedLinesFileReader; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Triple; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.text.MessageFormat; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary.*; - -public class EngineConnLogOperator implements Operator { - private static final Logger logger = LoggerFactory.getLogger(EngineConnLogOperator.class); - - public static final String OPERATOR_NAME = "engineConnLog"; - public static final CommonVars LOG_FILE_NAME = - CommonVars.apply("linkis.engineconn.log.filename", "stdout"); - public static final CommonVars MAX_LOG_FETCH_SIZE = - CommonVars.apply("linkis.engineconn.log.fetch.lines.max", 5000); - public static final CommonVars MAX_LOG_TAIL_START_SIZE = - CommonVars.apply("linkis.engineconn.log.tail.start.size"); - public static final CommonVars MULTILINE_PATTERN = - CommonVars.apply( - "linkis.engineconn.log.multiline.pattern", - "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}"); - public static final CommonVars MULTILINE_MAX = - CommonVars.apply("linkis.engineconn.log.multiline.max", 500); - - @Override - public String[] getNames() { - return new String[] {OPERATOR_NAME}; - } - - @Override - public Map apply(Map parameters) { - File logPath = getLogPath(parameters); - int lastRows = getAs(parameters, "lastRows", 0); - int pageSize = getAs(parameters, "pageSize", 100); - int fromLine = getAs(parameters, "fromLine", 1); - boolean enableTail = getAs(parameters, "enableTail", false); - if (lastRows > EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue()) { - throw new ECMErrorException( - CANNOT_FETCH_MORE_THAN.getErrorCode(), - MessageFormat.format( - CANNOT_FETCH_MORE_THAN.getErrorDesc(), - EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue().toString())); - } else if (lastRows > 0) { - String logs = Utils.exec(new String[] {"tail", "-n", lastRows + "", logPath.getPath()}, 5000); - Map stringObjectHashMap = new HashMap<>(); - stringObjectHashMap.put("logs", logs.split("\n")); - stringObjectHashMap.put("rows", logs.length()); - return stringObjectHashMap; - } - - String ignoreKeywords = getAs(parameters, "ignoreKeywords", ""); - String[] ignoreKeywordList = - StringUtils.isNotEmpty(ignoreKeywords) ? ignoreKeywords.split(",") : new String[0]; - - String onlyKeywords = getAs(parameters, "onlyKeywords", ""); - String[] onlyKeywordList = - StringUtils.isNotEmpty(onlyKeywords) ? onlyKeywords.split(",") : new String[0]; - - RandomAccessFile randomReader = null; - ReversedLinesFileReader reversedReader = null; - try { - if (enableTail) { - logger.info("enable log operator from tail to read"); - reversedReader = new ReversedLinesFileReader(logPath, Charset.defaultCharset()); - } else { - randomReader = new RandomAccessFile(logPath, "r"); - } - - ArrayList logs = new ArrayList<>(pageSize); - int readLine = 0, skippedLine = 0, lineNum = 0; - boolean rowIgnore = false; - int ignoreLine = 0; - Pattern linePattern = Pattern.compile(EngineConnLogOperator.MULTILINE_PATTERN.getValue()); - - int maxMultiline = MULTILINE_MAX.getValue(); - String line = randomAndReversedReadLine(randomReader, reversedReader); - - while (readLine < pageSize && line != null) { - lineNum += 1; - if (skippedLine < fromLine - 1) { - skippedLine += 1; - } else { - if (rowIgnore) { - Matcher matcher = linePattern.matcher(line); - if (matcher.matches()) { - ignoreLine = 0; - rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList); - } else { - ignoreLine += 1; - if (ignoreLine >= maxMultiline) { - rowIgnore = false; - } - } - if (!matcher.matches()) { - rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList); - } - } else { - rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList); - } - if (!rowIgnore) { - logs.add(line); - readLine += 1; - } - } - line = randomAndReversedReadLine(randomReader, reversedReader); - } - - if (enableTail) { - Collections.reverse(logs); - } - - Map resultMap = new HashMap<>(); - resultMap.put("logPath", logPath.getPath()); - resultMap.put("logs", logs); - resultMap.put("endLine", lineNum); - resultMap.put("rows", readLine); - return resultMap; - } catch (IOException e) { - logger.info("EngineConnLogOperator apply failed", e); - throw new ECMErrorException( - LOG_IS_NOT_EXISTS.getErrorCode(), LOG_IS_NOT_EXISTS.getErrorDesc()); - } finally { - IOUtils.closeQuietly(randomReader); - IOUtils.closeQuietly(reversedReader); - } - } - - private String randomAndReversedReadLine( - RandomAccessFile randomReader, ReversedLinesFileReader reversedReader) throws IOException { - if (randomReader != null) { - String line = randomReader.readLine(); - if (line != null) { - return new String(line.getBytes(StandardCharsets.ISO_8859_1), Charset.defaultCharset()); - } else { - return null; - } - } else { - return reversedReader.readLine(); - } - } - - protected File getLogPath(Map parameters) { - String logType = getAs(parameters, "logType", EngineConnLogOperator.LOG_FILE_NAME.getValue()); - - Triple engineConnInfo = getEngineConnInfo(parameters); - String engineConnLogDir = engineConnInfo.getLeft(); - String engineConnInstance = engineConnInfo.getMiddle(); - String ticketId = engineConnInfo.getRight(); - - File logPath = new File(engineConnLogDir, logType); - if (!logPath.exists() || !logPath.isFile()) { - throw new ECMErrorException( - LOGFILE_IS_NOT_EXISTS.getErrorCode(), - MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc(), logPath.toString())); - } - logger.info( - String.format( - "Try to fetch EngineConn(id: %s, instance: %s) logs from %s.", - ticketId, engineConnInstance, logPath.getPath())); - return logPath; - } - - protected Triple getEngineConnInfo(Map parameters) { - String logDIrSuffix = getAs(parameters, "logDirSuffix", ""); - String engineConnLogDir = - ECMConfiguration.ENGINECONN_ROOT_DIR() + File.separator + logDIrSuffix; - String ticketId = getAs(parameters, "ticketId", ""); - String engineConnInstance = ""; - return Triple.of(engineConnLogDir, engineConnInstance, ticketId); - } - - private boolean includeLine(String line, String[] onlyKeywordList, String[] ignoreKeywordList) { - boolean accept = - ignoreKeywordList.length == 0 || !Arrays.stream(ignoreKeywordList).anyMatch(line::contains); - if (accept) { - accept = - onlyKeywordList.length == 0 || Arrays.stream(onlyKeywordList).anyMatch(line::contains); - } - return accept; - } -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java deleted file mode 100644 index 6d3548274c..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.operator; - -import org.apache.linkis.common.exception.WarnException; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary; -import org.apache.linkis.ecm.server.exception.ECMErrorException; -import org.apache.linkis.ecm.server.exception.ECMWarnException; - -import org.apache.commons.lang3.tuple.Triple; - -import java.io.File; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary.LOG_IS_NOT_EXISTS; - -public class EngineConnYarnLogOperator extends EngineConnLogOperator { - private static final Logger logger = LoggerFactory.getLogger(EngineConnYarnLogOperator.class); - - private static final String YARN_LOG_OPERATOR_NAME = "engineConnYarnLog"; - - @Override - public String[] getNames() { - return new String[] {EngineConnYarnLogOperator.YARN_LOG_OPERATOR_NAME}; - } - - @Override - public Map apply(Map parameters) { - Map result = new HashMap<>(); - try { - result = super.apply(parameters); - } finally { - Object logPath = result.get("logPath"); - if (logPath instanceof String) { - File logFile = new File((String) logPath); - if (logFile.exists() && logFile.getName().startsWith(".")) { - // If is a temporary file, drop it - logger.info(String.format("Delete the temporary yarn log file: [%s]", logPath)); - if (!logFile.delete()) { - logger.warn(String.format("Fail to delete the temporary yarn log file: [%s]", logPath)); - } - } - } - } - return result; - } - - @Override - public File getLogPath(Map parameters) { - String ticketId, engineConnInstance, engineConnLogDir; - Triple engineConnInfo = getEngineConnInfo(parameters); - ticketId = engineConnInfo.getRight(); - engineConnInstance = engineConnInfo.getMiddle(); - engineConnLogDir = engineConnInfo.getLeft(); - - File rootLogDir = new File(engineConnLogDir); - if (!rootLogDir.exists() || !rootLogDir.isDirectory()) { - throw new ECMWarnException( - LOG_IS_NOT_EXISTS.getErrorCode(), - MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc(), rootLogDir)); - } - - String creator = getAsThrow(parameters, "creator"); - String applicationId = getAsThrow(parameters, "yarnApplicationId"); - File logPath = new File(engineConnLogDir, "yarn_" + applicationId); - if (!logPath.exists()) { - String tempLogFile = - String.format( - ".yarn_%s_%d_%d", - applicationId, System.currentTimeMillis(), Thread.currentThread().getId()); - try { - String command = - String.format( - "yarn logs -applicationId %s >> %s/%s", applicationId, rootLogDir, tempLogFile); - logger.info(String.format("Fetch yarn logs to temporary file: [%s]", command)); - - ProcessBuilder processBuilder = new ProcessBuilder(sudoCommands(creator, command)); - processBuilder.environment().putAll(System.getenv()); - processBuilder.redirectErrorStream(false); - Process process = processBuilder.start(); - boolean waitFor = process.waitFor(5, TimeUnit.SECONDS); - logger.trace(String.format("waitFor: %b, result: %d", waitFor, process.exitValue())); - if (waitFor && process.waitFor() == 0) { - command = - String.format( - "mv %s/%s %s/yarn_%s", rootLogDir, tempLogFile, rootLogDir, applicationId); - logger.info(String.format("Move and save yarn logs: [%s]", command)); - Utils.exec(sudoCommands(creator, command)); - } else { - logPath = new File(engineConnLogDir, tempLogFile); - if (!logPath.exists()) { - throw new WarnException( - -1, - String.format( - "Fetch yarn logs timeout, log aggregation has not completed or is not enabled")); - } - } - } catch (Exception e) { - throw new WarnException( - -1, - String.format( - "Fail to fetch yarn logs application: %s, message: %s", - applicationId, e.getMessage())); - } - } - if (!logPath.exists() || !logPath.isFile()) { - throw new ECMErrorException( - EngineconnServerErrorCodeSummary.LOGFILE_IS_NOT_EXISTS.getErrorCode(), - MessageFormat.format( - EngineconnServerErrorCodeSummary.LOGFILE_IS_NOT_EXISTS.getErrorDesc(), logPath)); - } - logger.info( - String.format( - "Try to fetch EngineConn(id: %s, instance: %s) yarn logs from %s in application id: %s", - ticketId, engineConnInstance, logPath.getPath(), applicationId)); - - return logPath; - } - - private String[] sudoCommands(String creator, String command) { - return new String[] { - "/bin/bash", - "-c", - "sudo su " + creator + " -c \"source ~/.bashrc 2>/dev/null; " + command + "\"" - }; - } -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java index 5a0ade21dc..2e351b00df 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -21,6 +21,7 @@ import org.apache.linkis.common.utils.Utils; import org.apache.linkis.ecm.server.conf.ECMConfiguration; import org.apache.linkis.ecm.server.service.EngineConnKillService; +import org.apache.linkis.ecm.utils.ECMCacheUtils; import org.apache.linkis.engineconn.common.conf.EngineConnConf; import org.apache.linkis.governance.common.utils.GovernanceUtils; import org.apache.linkis.manager.common.constant.AMConstant; @@ -61,6 +62,7 @@ public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest String pid = null; if (AMConstant.PROCESS_MARK.equals(engineStopRequest.getIdentifierType()) && StringUtils.isNotBlank(engineStopRequest.getIdentifier())) { + ECMCacheUtils.putStopECToCache(engineStopRequest.getServiceInstance(), engineStopRequest); pid = engineStopRequest.getIdentifier(); } logger.info("dealEngineConnStop return pid: {}", pid); @@ -189,13 +191,11 @@ private String getYarnAppRegexByEngineType(String engineType) { case "sqoop": regex = EngineConnConf.SQOOP_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); break; + case "seatunnel": case "flink": case "hive": regex = EngineConnConf.HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); break; - case "seatunnel": - regex = EngineConnConf.SEATUNNEL_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); - break; default: regex = ""; } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/LabelScoreServiceInstance.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/utils/ECMCacheUtils.java similarity index 53% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/LabelScoreServiceInstance.java rename to linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/utils/ECMCacheUtils.java index 8e5b57678a..52d140055d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/LabelScoreServiceInstance.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/utils/ECMCacheUtils.java @@ -15,37 +15,26 @@ * limitations under the License. */ -package org.apache.linkis.manager.label.score; +package org.apache.linkis.ecm.utils; import org.apache.linkis.common.ServiceInstance; -import org.apache.linkis.manager.common.entity.node.ScoreServiceInstance; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; -public class LabelScoreServiceInstance implements ScoreServiceInstance { +import java.util.concurrent.TimeUnit; - private double score = 0d; - private ServiceInstance serviceInstance; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; - public LabelScoreServiceInstance(ServiceInstance instance) { - this.serviceInstance = instance; - } - - @Override - public double getScore() { - return score; - } - - @Override - public void setScore(double score) { - this.score = score; - } +public class ECMCacheUtils { + private static Cache ecStopRequestCache = + CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.SECONDS).build(); - @Override - public ServiceInstance getServiceInstance() { - return serviceInstance; + public static void putStopECToCache( + ServiceInstance serviceInstance, EngineStopRequest engineStopRequest) { + ecStopRequestCache.put(serviceInstance, engineStopRequest); } - @Override - public void setServiceInstance(ServiceInstance serviceInstance) { - this.serviceInstance = serviceInstance; + public static EngineStopRequest getStopEC(ServiceInstance serviceInstance) { + return ecStopRequestCache.getIfPresent(serviceInstance); } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala index 4c02bfff05..dbd24e3562 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala @@ -20,7 +20,6 @@ package org.apache.linkis.ecm.server.conf import org.apache.linkis.common.conf.{CommonVars, TimeType} import org.apache.linkis.common.utils.ByteTimeUtils import org.apache.linkis.governance.common.conf.GovernanceCommonConf -import org.apache.linkis.manager.common.conf.RMConfiguration import java.io.File import java.util.concurrent.TimeUnit @@ -116,16 +115,8 @@ object ECMConfiguration { val ECM_PROCESS_SCRIPT_KILL: Boolean = CommonVars[Boolean]("wds.linkis.ecm.script.kill.engineconn", true).getValue - val ECM_YARN_CLUSTER_NAME: String = - CommonVars( - "wds.linkis.ecm.yarn.cluster.name", - RMConfiguration.DEFAULT_YARN_CLUSTER_NAME.getValue - ).getValue - - val ECM_YARN_CLUSTER_TYPE: String = - CommonVars( - "wds.linkis.ecm.yarn.cluster.type", - RMConfiguration.DEFAULT_YARN_TYPE.getValue - ).getValue - + val EC_CAN_RETRY_EXIT_CODES: Array[Int] = + CommonVars[String]("linkis.ecm.can.retry.exit.codes", "143").getValue + .split(",") + .map(_.toInt); } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorException.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorException.scala index 1b397d04e5..e50b2ff84f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorException.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorException.scala @@ -17,22 +17,15 @@ package org.apache.linkis.ecm.server.exception -import org.apache.linkis.common.exception.{ - ErrorException, - ExceptionLevel, - LinkisRuntimeException, - WarnException -} +import org.apache.linkis.common.exception.{ErrorException, WarnException} -class ECMErrorException(errorCode: Int, desc: String) - extends LinkisRuntimeException(errorCode, desc) { +class ECMErrorException(errorCode: Int, desc: String) extends ErrorException(errorCode, desc) { def this(errorCode: Int, desc: String, t: Throwable) { this(errorCode, desc) this.initCause(t) } - override def getLevel: ExceptionLevel = ExceptionLevel.ERROR } class ECMWarnException(errCode: Int, desc: String) extends WarnException(errCode, desc) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala index 39efe74cc8..26cccfc9c3 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala @@ -113,6 +113,7 @@ class JarUDFLoadECMHook extends ECMHook with Logging { ) } } + // LaunchConstants.addPathToClassPath(pel.environment, udfDir + File.separator + "*") val udfJars = fileNameSet.map(udfDir + File.separator + _) pel.environment.put(Environment.UDF_JARS.toString, udfJars.mkString(",")); } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala new file mode 100644 index 0000000000..66327dadcf --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.operator + +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ +import org.apache.linkis.ecm.server.conf.ECMConfiguration +import org.apache.linkis.ecm.server.exception.ECMErrorException +import org.apache.linkis.ecm.server.service.LocalDirsHandleService +import org.apache.linkis.manager.common.operator.Operator + +import org.apache.commons.io.IOUtils +import org.apache.commons.io.input.ReversedLinesFileReader +import org.apache.commons.lang3.StringUtils + +import java.io.{File, RandomAccessFile} +import java.nio.charset.{Charset, StandardCharsets} +import java.text.MessageFormat +import java.util +import java.util.Collections + +import scala.util.matching.Regex + +class EngineConnLogOperator extends Operator with Logging { + + override def getNames: Array[String] = Array(EngineConnLogOperator.OPERATOR_NAME) + + override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { + val logPath = getLogPath + val lastRows = getAs("lastRows", 0) + val pageSize = getAs("pageSize", 100) + val fromLine = getAs("fromLine", 1) + val enableTail = getAs("enableTail", false) + if (lastRows > EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue) { + throw new ECMErrorException( + CANNOT_FETCH_MORE_THAN.getErrorCode, + MessageFormat.format( + CANNOT_FETCH_MORE_THAN.getErrorDesc, + EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue.toString + ) + ) + } else if (lastRows > 0) { + val logs = Utils.exec(Array("tail", "-n", lastRows + "", logPath.getPath), 5000).split("\n") + return Map("logs" -> logs, "rows" -> logs.length) + } + + val ignoreKeywords = getAs("ignoreKeywords", "") + val ignoreKeywordList = + if (StringUtils.isNotEmpty(ignoreKeywords)) ignoreKeywords.split(",") + else Array.empty[String] + val onlyKeywords = getAs("onlyKeywords", "") + val onlyKeywordList = + if (StringUtils.isNotEmpty(onlyKeywords)) onlyKeywords.split(",") else Array.empty[String] + var randomReader: RandomAccessFile = null + var reversedReader: ReversedLinesFileReader = null + if (enableTail) { + logger.info("enable log operator from tail to read") + reversedReader = new ReversedLinesFileReader(logPath, Charset.defaultCharset()) + } else { + randomReader = new RandomAccessFile(logPath, "r") + } + def randomAndReversedReadLine(): String = { + if (null != randomReader) { + val line = randomReader.readLine() + if (line != null) { + new String(line.getBytes(StandardCharsets.ISO_8859_1), Charset.defaultCharset()) + } else null + } else { + reversedReader.readLine() + } + } + val logs = new util.ArrayList[String](pageSize) + var readLine, skippedLine, lineNum = 0 + var rowIgnore = false + var ignoreLine = 0 + val linePattern = getLinePattern + val maxMultiline = EngineConnLogOperator.MULTILINE_MAX.getValue + Utils.tryFinally { + var line = randomAndReversedReadLine() + while (readLine < pageSize && line != null) { + lineNum += 1 + if (skippedLine < fromLine - 1) { + skippedLine += 1 + } else { + if (rowIgnore) { + linePattern match { + case reg: Regex => + if (reg.findFirstIn(line).isDefined) { + ignoreLine = 0 + rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) + } else { + ignoreLine += 1 + if (ignoreLine >= maxMultiline) { + rowIgnore = false + } + } + case _ => rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) + } + } else { + rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) + } + if (!rowIgnore) { + logs.add(line) + readLine += 1 + } + } + line = randomAndReversedReadLine() + } + } { + IOUtils.closeQuietly(randomReader) + IOUtils.closeQuietly(reversedReader) + } + if (enableTail) Collections.reverse(logs) + Map("logPath" -> logPath.getPath, "logs" -> logs, "endLine" -> lineNum, "rows" -> readLine) + } + + protected def getLogPath(implicit parameters: Map[String, Any]): File = { + val (ticketId, engineConnInstance, engineConnLogDir) = getEngineConnInfo(parameters) + val logPath = + new File(engineConnLogDir, getAs("logType", EngineConnLogOperator.LOG_FILE_NAME.getValue)); + if (!logPath.exists() || !logPath.isFile) { + throw new ECMErrorException( + LOGFILE_IS_NOT_EXISTS.getErrorCode, + MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath) + ) + } + logger.info( + s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) logs from ${logPath.getPath}." + ) + logPath + } + + protected def getLinePattern: Regex = { + Option(EngineConnLogOperator.MULTILINE_PATTERN.getValue) match { + case Some(pattern) => pattern.r + case _ => null + } + } + + protected def getEngineConnInfo(implicit + parameters: Map[String, Any] + ): (String, String, String) = { + val logDIrSuffix = getAs("logDirSuffix", "") + val ecLogPath = ECMConfiguration.ENGINECONN_ROOT_DIR + File.separator + logDIrSuffix + val ticketId = getAs("ticketId", "") + (ticketId, "", ecLogPath) + } + + private def includeLine( + line: String, + onlyKeywordList: Array[String], + ignoreKeywordList: Array[String] + ): Boolean = { + var accept: Boolean = ignoreKeywordList.isEmpty || !ignoreKeywordList.exists(line.contains) + if (accept) { + accept = onlyKeywordList.isEmpty || onlyKeywordList.exists(line.contains) + } + accept + } + +} + +object EngineConnLogOperator { + val OPERATOR_NAME = "engineConnLog" + val LOG_FILE_NAME = CommonVars("linkis.engineconn.log.filename", "stdout") + val MAX_LOG_FETCH_SIZE = CommonVars("linkis.engineconn.log.fetch.lines.max", 5000) + + val MAX_LOG_TAIL_START_SIZE = CommonVars("linkis.engineconn.log.tail.start.size", 20000) + + // yyyy-MM-dd HH:mm:ss.SSS + val MULTILINE_PATTERN = CommonVars( + "linkis.engineconn.log.multiline.pattern", + "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}" + ) + + val MULTILINE_MAX = CommonVars("linkis.engineconn.log.multiline.max", 500) +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala new file mode 100644 index 0000000000..2ef8be706c --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.operator + +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.io.FsPath +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ +import org.apache.linkis.ecm.server.exception.ECMErrorException +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.utils.StorageUtils + +import org.apache.commons.lang3.StringUtils + +import java.io.File +import java.text.MessageFormat +import java.util +import java.util.concurrent.{Callable, ConcurrentHashMap, ExecutorService, Future, TimeUnit} + +import scala.collection.JavaConverters._ +import scala.util.matching.Regex + +class EngineConnYarnLogOperator extends EngineConnLogOperator { + + private implicit val fs: FileSystem = + FSFactory.getFs(StorageUtils.FILE).asInstanceOf[FileSystem] + + /** + * Yarn log fetchers + */ + private def yarnLogFetchers: ConcurrentHashMap[String, Future[String]] = + new ConcurrentHashMap[String, Future[String]]() + + override def getNames: Array[String] = Array(EngineConnYarnLogOperator.OPERATOR_NAME) + + override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { + var result: Map[String, Any] = Map() + Utils.tryFinally { + result = super.apply(parameters) + result + } {} + } + + override def getLogPath(parameters: Map[String, Any]): File = { + val applicationId = getAsThrow[String]("yarnApplicationId", parameters) + val (ticketId, engineConnInstance, engineConnLogDir) = getEngineConnInfo(parameters) + val rootLogPath = EngineConnYarnLogOperator.YARN_LOG_STORAGE_PATH.getValue match { + case storePath if StringUtils.isNotBlank(storePath) => + val logPath = new FsPath(StorageUtils.FILE_SCHEMA + storePath + "/" + applicationId) + // Invoke to create directory + fs.mkdir(logPath) + // 777 permission + fs.setPermission(logPath, "rwxrwxrwx") + logPath + case _ => new FsPath(StorageUtils.FILE_SCHEMA + engineConnLogDir) + } + if (!fs.exists(rootLogPath) || !rootLogPath.toFile.isDirectory) { + throw new ECMErrorException( + LOG_IS_NOT_EXISTS.getErrorCode, + MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc, rootLogPath.getPath) + ) + } + val creator = getAsThrow[String]("creator", parameters) + var logPath = new FsPath( + StorageUtils.FILE_SCHEMA + rootLogPath.getPath + "/yarn_" + applicationId + ) + if (!fs.exists(logPath)) { + val fetcher = yarnLogFetchers.computeIfAbsent( + applicationId, + new util.function.Function[String, Future[String]] { + override def apply(v1: String): Future[String] = + requestToFetchYarnLogs(creator, applicationId, rootLogPath.getPath) + } + ) + // Just wait 5 seconds + Option(fetcher.get(5, TimeUnit.SECONDS)) match { + case Some(path) => logPath = new FsPath(StorageUtils.FILE_SCHEMA + path) + case _ => + } + + } + if (!fs.exists(logPath) || logPath.toFile.isDirectory) { + throw new ECMErrorException( + LOGFILE_IS_NOT_EXISTS.getErrorCode, + MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath.getPath) + ) + } + logger.info( + s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) yarn logs from ${logPath.getPath} in application id: $applicationId" + ) + logPath.toFile + } + + /** + * Not support line pattern in yarn log + * @return + */ + override protected def getLinePattern: Regex = null + + /** + * Request the log fetcher + * + * @param creator + * creator + * @param applicationId + * application id + * @param logPath + * log path + * @return + */ + private def requestToFetchYarnLogs( + creator: String, + applicationId: String, + yarnLogDir: String + ): Future[String] = { + EngineConnYarnLogOperator.YARN_LOG_FETCH_SCHEDULER.submit(new Callable[String] { + override def call(): String = { + val logPath = new FsPath(StorageUtils.FILE_SCHEMA + yarnLogDir + "/yarn_" + applicationId) + if (!fs.exists(logPath)) { + val tempLogFile = + s".yarn_${applicationId}_${System.currentTimeMillis()}_${Thread.currentThread().getId}" + Utils.tryCatch { + var command = + s"yarn logs -applicationId $applicationId >> $yarnLogDir/$tempLogFile" + logger.info(s"Fetch yarn logs to temporary file: [$command]") + val processBuilder = new ProcessBuilder(sudoCommands(creator, command): _*) + processBuilder.environment.putAll(sys.env.asJava) + processBuilder.redirectErrorStream(false) + val process = processBuilder.start() + val exitCode = process.waitFor() + logger.trace(s"Finish to fetch yan logs to temporary file, result: ${exitCode}") + if (exitCode == 0) { + command = s"mv $yarnLogDir/$tempLogFile $yarnLogDir/yarn_$applicationId" + logger.info(s"Move and save yarn logs(${applicationId}): [$command]") + Utils.exec(sudoCommands(creator, command)) + } + } { e: Throwable => + logger.error( + s"Fail to fetch yarn logs application: $applicationId, message: ${e.getMessage}" + ) + } + val tmpFile = new File(yarnLogDir, tempLogFile) + if (tmpFile.exists()) { + logger.info(s"Delete temporary file: [${tempLogFile}] in yarn logs fetcher") + tmpFile.delete() + } + } + // Remove future + yarnLogFetchers.remove(applicationId) + if (fs.exists(logPath)) logPath.getPath else null + } + }) + } + + private def sudoCommands(creator: String, command: String): Array[String] = { + Array( + "/bin/bash", + "-c", + "sudo su " + creator + " -c \"source ~/.bashrc 2>/dev/null; " + command + "\"" + ) + } + +} + +object EngineConnYarnLogOperator { + val OPERATOR_NAME = "engineConnYarnLog" + + // Specific the path to store the yarn logs + val YARN_LOG_STORAGE_PATH: CommonVars[String] = + CommonVars("linkis.engineconn.log.yarn.storage-path", "") + + val YARN_LOG_FETCH_THREAD: CommonVars[Int] = + CommonVars("linkis.engineconn.log.yarn.fetch.thread-num", 5) + + val YARN_LOG_FETCH_SCHEDULER: ExecutorService = + Utils.newFixedThreadPool(YARN_LOG_FETCH_THREAD.getValue + 1, "yarn_logs_fetch", false) + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/restful/ListenerTest.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMMetricsService.scala similarity index 91% rename from linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/restful/ListenerTest.java rename to linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMMetricsService.scala index 11a592924a..62ae7c1fea 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/test/java/org/apache/linkis/cs/client/test/restful/ListenerTest.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMMetricsService.scala @@ -15,6 +15,6 @@ * limitations under the License. */ -package org.apache.linkis.cs.client.test.restful; +package org.apache.linkis.ecm.server.service -public class ListenerTest {} +trait ECMMetricsService {} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index 390822df0d..df00ed4960 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -121,11 +121,10 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w Sender .getSender(MANAGER_SERVICE_NAME) .send( - new EngineConnStatusCallbackToAM( + EngineConnStatusCallbackToAM( conn.getServiceInstance, NodeStatus.Failed, - " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t), - true + " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t) ) ) conn.setStatus(NodeStatus.Failed) @@ -147,12 +146,10 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w throw t } LoggerUtils.removeJobIdMDC() - val label = LabelUtil.getEngingeConnRuntimeModeLabel(request.labels) val isYarnClusterMode: Boolean = if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) true else false - val engineNode = new AMEngineNode() engineNode.setLabels(conn.getLabels) engineNode.setServiceInstance(conn.getServiceInstance) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala index e334a8ac29..ed1490294c 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala @@ -120,6 +120,8 @@ class BmlResourceLocalizationService extends ResourceLocalizationService with Lo } } + private val bmlResourceSuffix = ".zip" + private def createDirIfNotExit(noSchemaPath: String): String = { val fsPath = new FsPath(schema + noSchemaPath) if (!fs.exists(fsPath)) { diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMMetricsService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMMetricsService.scala new file mode 100644 index 0000000000..5d7da6b91e --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMMetricsService.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.service.impl + +import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener} +import org.apache.linkis.ecm.server.service.ECMMetricsService + +class DefaultECMMetricsService extends ECMMetricsService with ECMEventListener { + override def onEvent(event: ECMEvent): Unit = {} +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala index 51f223e476..4c7807dad1 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala @@ -17,7 +17,6 @@ package org.apache.linkis.ecm.server.service.impl -import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener} import org.apache.linkis.ecm.server.conf.ECMConfiguration._ @@ -31,7 +30,6 @@ import org.apache.linkis.manager.common.protocol.em.{ StopEMRequest } import org.apache.linkis.manager.label.constant.LabelKeyConstant -import org.apache.linkis.manager.label.entity.SerializableLabel import org.apache.linkis.rpc.Sender import java.util @@ -52,25 +50,17 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener request } - def getLabelsFromArgs(params: Array[String]): util.Map[String, AnyRef] = { + private def getLabelsFromArgs(params: Array[String]): util.Map[String, AnyRef] = { import scala.collection.JavaConverters._ - val labelRegex = """label\.(.+)\.(.+)=(.+)""".r val labels = new util.HashMap[String, AnyRef]() - // TODO: magic labels.asScala += LabelKeyConstant.SERVER_ALIAS_KEY -> Collections.singletonMap( "alias", ENGINE_CONN_MANAGER_SPRING_NAME ) - - if (Configuration.IS_MULTIPLE_YARN_CLUSTER) { - labels.asScala += LabelKeyConstant.YARN_CLUSTER_KEY -> - (ECM_YARN_CLUSTER_TYPE + "_" + ECM_YARN_CLUSTER_NAME) - } - // TODO: group by key labels } - def getEMRegiterResourceFromConfiguration: NodeResource = { + private def getEMRegiterResourceFromConfiguration: NodeResource = { val maxResource = new LoadInstanceResource( ECMUtils.inferDefaultMemory(), ECM_MAX_CORES_AVAILABLE, @@ -107,11 +97,9 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener logger.info("start register ecm") val response = Sender.getSender(MANAGER_SERVICE_NAME).ask(request) response match { - case registerEMResponse: RegisterEMResponse => - if (!registerEMResponse.getIsSuccess) { - logger.error( - s"Failed to register info to linkis manager, reason: ${registerEMResponse.getMsg}" - ) + case RegisterEMResponse(isSuccess, msg) => + if (!isSuccess) { + logger.error(s"Failed to register info to linkis manager, reason: $msg") System.exit(1) } case _ => diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala index 12b813d67f..3a541df8b9 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala @@ -27,34 +27,26 @@ import org.apache.commons.lang3.exception.ExceptionUtils import org.springframework.stereotype.Service -import java.util +import scala.collection.JavaConverters.mapAsScalaMapConverter @Service class DefaultOperateService extends OperateService with Logging { @Receiver override def executeOperation(ecmOperateRequest: ECMOperateRequest): ECMOperateResponse = { - val parameters = ecmOperateRequest.getParameters() - val operator = Utils.tryCatch(OperatorFactory.apply().getOperatorRequest(parameters)) { t => - logger.error(s"Get operator failed, parameters is ${ecmOperateRequest.getParameters}.", t) - return new ECMOperateResponse( - new util.HashMap[String, Object](), - true, - ExceptionUtils.getRootCauseMessage(t) - ) + val parameters = ecmOperateRequest.parameters.asScala.toMap + val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)) { t => + logger.error(s"Get operator failed, parameters is ${ecmOperateRequest.parameters}.", t) + return ECMOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) } logger.info( - s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${ecmOperateRequest.getParameters}." + s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${ecmOperateRequest.parameters}." ) val result = Utils.tryCatch(operator(parameters)) { t => logger.error(s"Execute ${operator.getClass.getSimpleName} failed.", t) - return new ECMOperateResponse( - new util.HashMap[String, Object](), - true, - ExceptionUtils.getRootCauseMessage(t) - ) + return ECMOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) } - new ECMOperateResponse(result) + ECMOperateResponse(result) } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index 360bca269d..b20590f04a 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -26,6 +26,7 @@ import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SERVICE_NAME import org.apache.linkis.ecm.server.listener.EngineConnStopEvent import org.apache.linkis.ecm.server.service.LocalDirsHandleService +import org.apache.linkis.ecm.utils.ECMCacheUtils import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.enumeration.NodeStatus @@ -90,20 +91,36 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe ) } if (exitCode != 0) { - val canRetry = if (errorMsg.isEmpty) true else false - logger.warn( - s"Failed to start ec ${engineConn.getServiceInstance}, status shutting down exit code ${exitCode}, canRetry ${canRetry}, logPath ${logPath}" - ) - Sender - .getSender(MANAGER_SERVICE_NAME) - .send( - new EngineConnStatusCallbackToAM( - engineConn.getServiceInstance, - NodeStatus.ShuttingDown, - "Failed to start EngineConn, reason: " + errorMsg + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help", - canRetry + val stopRequest = ECMCacheUtils.getStopEC(engineConn.getServiceInstance) + if ( + null != stopRequest && engineConn.getPid != null && engineConn.getPid.equals( + stopRequest.getIdentifier ) + ) { + logger.info( + s"EC ${engineConn.getServiceInstance} exit should by kill stop request $stopRequest, do not report status" + ) + } else { + val canRetry = + if (errorMsg.isEmpty || ECMConfiguration.EC_CAN_RETRY_EXIT_CODES.contains(exitCode)) { + true + } else { + false + } + logger.warn( + s"Failed to start ec ${engineConn.getServiceInstance}, status shutting down exit code ${exitCode}, canRetry ${canRetry}, logPath ${logPath}" ) + Sender + .getSender(MANAGER_SERVICE_NAME) + .send( + EngineConnStatusCallbackToAM( + engineConn.getServiceInstance, + NodeStatus.ShuttingDown, + "Failed to start EngineConn, reason: " + errorMsg + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help", + canRetry + ) + ) + } engineConn.setStatus(NodeStatus.ShuttingDown) } else { engineConn.setStatus(NodeStatus.Success) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/test/java/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterServiceTest.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/test/java/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterServiceTest.java deleted file mode 100644 index f65fd68780..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/test/java/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterServiceTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl; - -import org.apache.linkis.common.ServiceInstance; -import org.apache.linkis.manager.common.protocol.em.RegisterEMRequest; -import org.apache.linkis.manager.label.constant.LabelKeyConstant; -import org.apache.linkis.rpc.serializer.ProtostuffSerializeUtil; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import org.junit.jupiter.api.Test; - -import static org.apache.linkis.manager.label.conf.LabelCommonConfig.ENGINE_CONN_MANAGER_SPRING_NAME; - -public class DefaultECMRegisterServiceTest { - @Test - void testECM() { - DefaultECMRegisterService defaultECMRegisterService = new DefaultECMRegisterService(); - RegisterEMRequest request = new RegisterEMRequest(); - ServiceInstance instance = new ServiceInstance(); - instance.setInstance("127.0.0.1:9001"); - instance.setApplicationName("ecm"); - request.setUser("hadoop"); - request.setServiceInstance(instance); - request.setAlias(instance.getApplicationName()); - - Map labels = new HashMap<>(); - labels.put( - LabelKeyConstant.SERVER_ALIAS_KEY, - Collections.singletonMap("alias", ENGINE_CONN_MANAGER_SPRING_NAME)); - request.setLabels(defaultECMRegisterService.getLabelsFromArgs(null)); - // the ECMUtils.inferDefaultMemory() will throw error disable the test - // request.setNodeResource(defaultECMRegisterService.getEMRegiterResourceFromConfiguration()); - String res = ProtostuffSerializeUtil.serialize(request); - ProtostuffSerializeUtil.deserialize(res, RegisterEMRequest.class); - } -} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java deleted file mode 100644 index 3f98e6a988..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.once.executor.operator; - -import org.apache.linkis.engineconn.common.exception.EngineConnException; -import org.apache.linkis.engineconn.once.executor.OnceExecutor; -import org.apache.linkis.engineconn.once.executor.OperableOnceExecutor; -import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorManager$; -import org.apache.linkis.manager.common.operator.Operator; -import org.apache.linkis.manager.common.operator.OperatorFactory; -import org.apache.linkis.protocol.engine.JobProgressInfo; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class OperableOnceEngineConnOperator implements Operator { - - public static final String PROGRESS_OPERATOR_NAME = "engineConnProgress"; - - public static final String METRICS_OPERATOR_NAME = "engineConnMetrics"; - - public static final String DIAGNOSIS_OPERATOR_NAME = "engineConnDiagnosis"; - - @Override - public String[] getNames() { - return new String[] {PROGRESS_OPERATOR_NAME, METRICS_OPERATOR_NAME, DIAGNOSIS_OPERATOR_NAME}; - } - - @Override - public Map apply(Map parameters) { - String operatorName = OperatorFactory.apply().getOperatorName(parameters); - OnceExecutor reportExecutor = OnceExecutorManager$.MODULE$.getInstance().getReportExecutor(); - if (reportExecutor instanceof OperableOnceExecutor) { - OperableOnceExecutor operableOnceExecutor = (OperableOnceExecutor) reportExecutor; - switch (operatorName) { - case PROGRESS_OPERATOR_NAME: - List> progressInfoMap = new ArrayList<>(); - JobProgressInfo[] progressInfoList = operableOnceExecutor.getProgressInfo(); - if (progressInfoList != null && progressInfoList.length != 0) { - for (JobProgressInfo progressInfo : progressInfoList) { - Map infoMap = new HashMap<>(); - infoMap.put("id", progressInfo.id()); - infoMap.put("totalTasks", progressInfo.totalTasks()); - infoMap.put("runningTasks", progressInfo.runningTasks()); - infoMap.put("failedTasks", progressInfo.failedTasks()); - infoMap.put("succeedTasks", progressInfo.succeedTasks()); - progressInfoMap.add(infoMap); - } - } - Map resultMap = new HashMap<>(); - resultMap.put("progress", operableOnceExecutor.getProgress()); - resultMap.put("progressInfo", progressInfoMap); - return resultMap; - case METRICS_OPERATOR_NAME: - return new HashMap() { - { - put("metrics", operableOnceExecutor.getMetrics()); - } - }; - case DIAGNOSIS_OPERATOR_NAME: - return new HashMap() { - { - put("diagnosis", operableOnceExecutor.getDiagnosis()); - } - }; - default: - throw new EngineConnException( - 20308, "This engineConn don't support " + operatorName + " operator."); - } - } else { - throw new EngineConnException( - 20308, "This engineConn don't support " + operatorName + " operator."); - } - } -} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutor.scala index 30faaa8294..dee4a885dd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutor.scala @@ -186,10 +186,8 @@ trait ManageableOnceExecutor extends AccessibleExecutor with OnceExecutor with R logger.error(msg) Utils.tryFinally { this.ensureAvailable(transition(NodeStatus.Failed)) - } { close() - stopOnceExecutor(msg) - } + }(stopOnceExecutor(msg)) true } @@ -199,10 +197,8 @@ trait ManageableOnceExecutor extends AccessibleExecutor with OnceExecutor with R logger.warn(msg) Utils.tryFinally { this.ensureAvailable(transition(NodeStatus.Success)) - } { close() - stopOnceExecutor(msg) - } + }(stopOnceExecutor(msg)) true } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutorExecutionContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutorExecutionContext.scala index 5d66212069..c25d2f580c 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutorExecutionContext.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/OnceExecutorExecutionContext.scala @@ -22,7 +22,7 @@ import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetWriter} import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.executor.ExecutorExecutionContext import org.apache.linkis.governance.common.entity.job.OnceExecutorContent -import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetWriterFactory} +import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} class OnceExecutorExecutionContext( engineCreationContext: EngineCreationContext, @@ -40,14 +40,14 @@ class OnceExecutorExecutionContext( ): ResultSet[_ <: MetaData, _ <: Record] = resultSetFactory.getResultSetByType(resultSetType) - override protected def getDefaultResultSetByType: String = resultSetFactory.getResultSetType()(0) + override protected def getDefaultResultSetByType: String = resultSetFactory.getResultSetType(0) override protected def newResultSetWriter( resultSet: ResultSet[_ <: MetaData, _ <: Record], resultSetPath: FsPath, alias: String - ): org.apache.linkis.common.io.resultset.ResultSetWriter[_ <: MetaData, _ <: Record] = - ResultSetWriterFactory.getResultSetWriter( + ): ResultSetWriter[_ <: MetaData, _ <: Record] = + ResultSetWriter.getResultSetWriter( resultSet, 0, resultSetPath, diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala new file mode 100644 index 0000000000..ae1828d1c0 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.once.executor.operator + +import org.apache.linkis.engineconn.common.exception.EngineConnException +import org.apache.linkis.engineconn.once.executor.OperableOnceExecutor +import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorManager +import org.apache.linkis.manager.common.operator.{Operator, OperatorFactory} + +class OperableOnceEngineConnOperator extends Operator { + + import OperableOnceEngineConnOperator._ + + override def getNames: Array[String] = + Array(PROGRESS_OPERATOR_NAME, METRICS_OPERATOR_NAME, DIAGNOSIS_OPERATOR_NAME) + + override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { + val operatorName = OperatorFactory().getOperatorName(parameters) + OnceExecutorManager.getInstance.getReportExecutor match { + case operableOnceExecutor: OperableOnceExecutor => + operatorName match { + case PROGRESS_OPERATOR_NAME => + val progressInfo = operableOnceExecutor.getProgressInfo + val progressInfoMap = if (progressInfo != null && progressInfo.nonEmpty) { + progressInfo.map(progressInfo => + Map( + "id" -> progressInfo.id, + "totalTasks" -> progressInfo.totalTasks, + "runningTasks" -> progressInfo.runningTasks, + "failedTasks" -> progressInfo.failedTasks, + "succeedTasks" -> progressInfo.succeedTasks + ) + ) + } else Array.empty[Map[String, Any]] + Map("progress" -> operableOnceExecutor.getProgress, "progressInfo" -> progressInfoMap) + case METRICS_OPERATOR_NAME => + Map("metrics" -> operableOnceExecutor.getMetrics) + case DIAGNOSIS_OPERATOR_NAME => + Map("diagnosis" -> operableOnceExecutor.getDiagnosis) + case _ => + throw EngineConnException( + 20308, + s"This engineConn don't support $operatorName operator." + ) + } + case _ => + throw EngineConnException(20308, s"This engineConn don't support $operatorName operator.") + } + } + +} + +object OperableOnceEngineConnOperator { + val PROGRESS_OPERATOR_NAME = "engineConnProgress" + val METRICS_OPERATOR_NAME = "engineConnMetrics" + val DIAGNOSIS_OPERATOR_NAME = "engineConnDiagnosis" +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala index cf8b9c00b8..9af394da80 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala @@ -20,7 +20,10 @@ package org.apache.linkis.engineconn.computation.executor.async import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.Utils +import org.apache.linkis.engineconn.acessible.executor.info.DefaultNodeHealthyInfoManager import org.apache.linkis.engineconn.acessible.executor.listener.event.TaskResponseErrorEvent +import org.apache.linkis.engineconn.acessible.executor.utils.AccessibleExecutorUtils +import org.apache.linkis.engineconn.acessible.executor.utils.AccessibleExecutorUtils.currentEngineIsUnHealthy import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.apache.linkis.engineconn.computation.executor.execute.{ @@ -29,13 +32,16 @@ import org.apache.linkis.engineconn.computation.executor.execute.{ } import org.apache.linkis.engineconn.computation.executor.hook.ComputationExecutorHook import org.apache.linkis.engineconn.core.EngineConnObject +import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.entity.ConcurrentExecutor import org.apache.linkis.engineconn.executor.listener.{ EngineConnSyncListenerBus, ExecutorListenerBusContext } import org.apache.linkis.governance.common.entity.ExecutionNodeStatus -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} +import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus} +import org.apache.linkis.manager.label.entity.entrance.ExecuteOnceLabel import org.apache.linkis.protocol.engine.JobProgressInfo import org.apache.linkis.scheduler.executer._ import org.apache.linkis.scheduler.listener.JobListener @@ -48,6 +54,8 @@ import org.apache.commons.lang3.exception.ExceptionUtils import java.util import java.util.concurrent.ConcurrentHashMap +import DataWorkCloudApplication.getApplicationContext + abstract class AsyncConcurrentComputationExecutor(override val outputPrintLimit: Int = 1000) extends ComputationExecutor(outputPrintLimit) with ConcurrentExecutor @@ -97,9 +105,6 @@ abstract class AsyncConcurrentComputationExecutor(override val outputPrintLimit: }) } { e => logger.info("failed to do with hook", e) - engineExecutionContext.appendStdout( - LogUtils.generateWarn(s"failed execute hook: ${ExceptionUtils.getStackTrace(e)}") - ) } if (hookedCode.length > 100) { logger.info(s"hooked after code: ${hookedCode.substring(0, 100)} ....") @@ -207,6 +212,7 @@ abstract class AsyncConcurrentComputationExecutor(override val outputPrintLimit: s"Executor is busy but still got new task ! Running task num : ${getRunningTask}" ) } + runningTasks.increase() if (getRunningTask >= getConcurrentLimit) synchronized { if (getRunningTask >= getConcurrentLimit && NodeStatus.isIdle(getStatus)) { logger.info( @@ -215,13 +221,25 @@ abstract class AsyncConcurrentComputationExecutor(override val outputPrintLimit: transition(NodeStatus.Busy) } } - runningTasks.increase() } override def onJobCompleted(job: Job): Unit = { + runningTasks.decrease() job match { case asyncEngineConnJob: AsyncEngineConnJob => + val jobId = JobUtils.getJobIdFromMap(asyncEngineConnJob.getEngineConnTask.getProperties) + LoggerUtils.setJobIdMDC(jobId) + + if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) synchronized { + if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) { + logger.info( + s"running task($getRunningTask) < concurrent limit $getConcurrentLimit, now to mark engine to Unlock " + ) + transition(NodeStatus.Unlock) + } + } + job.getState match { case Succeed => succeedTasks.increase() @@ -241,22 +259,42 @@ abstract class AsyncConcurrentComputationExecutor(override val outputPrintLimit: } removeJob(asyncEngineConnJob.getEngineConnTask.getTaskId) clearTaskCache(asyncEngineConnJob.getEngineConnTask.getTaskId) - + // execute once should try to shutdown + if ( + asyncEngineConnJob.getEngineConnTask.getLables.exists(_.isInstanceOf[ExecuteOnceLabel]) + ) { + if (!hasTaskRunning()) { + logger.warn( + s"engineConnTask(${asyncEngineConnJob.getEngineConnTask.getTaskId}) is execute once, now to mark engine to Finished" + ) + ExecutorManager.getInstance.getReportExecutor.tryShutdown() + } + } + // unhealthy node should try to shutdown + if (!hasTaskRunning() && currentEngineIsUnHealthy()) { + logger.info( + s"engineConnTask(${asyncEngineConnJob.getEngineConnTask.getTaskId}) is unHealthy, now to mark engine to Finished" + ) + ExecutorManager.getInstance.getReportExecutor.tryShutdown() + } + LoggerUtils.setJobIdMDC(jobId) case _ => } - if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) synchronized { - if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) { - logger.info( - s"running task($getRunningTask) < concurrent limit $getConcurrentLimit, now to mark engine to Unlock " - ) - transition(NodeStatus.Unlock) - } - } } override def hasTaskRunning(): Boolean = { getRunningTask > 0 } + override def transition(toStatus: NodeStatus): Unit = { + if (getRunningTask >= getConcurrentLimit && NodeStatus.Unlock == toStatus) { + logger.info( + s"running task($getRunningTask) > concurrent limit:$getConcurrentLimit, can not to mark EC to Unlock" + ) + return + } + super.transition(toStatus) + } + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala index cb04f364fb..98f04daaa2 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineconn.computation.executor.bml import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} import org.apache.linkis.common.exception.ErrorException import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext import org.apache.linkis.engineconn.computation.executor.hook.ComputationExecutorHook @@ -62,6 +63,7 @@ class BmlEnginePreExecuteHook extends ComputationExecutorHook with Logging { ): String = { val props = engineExecutionContext.getProperties if (null != props && props.containsKey(GovernanceConstant.TASK_RESOURCES_STR)) { +// val workDir = ComputationEngineUtils.getCurrentWorkDir val jobId = engineExecutionContext.getJobId props.get(GovernanceConstant.TASK_RESOURCES_STR) match { case resources: util.List[Object] => @@ -71,6 +73,8 @@ class BmlEnginePreExecuteHook extends ComputationExecutorHook with Logging { val resourceId = resource.get(GovernanceConstant.TASK_RESOURCE_ID_STR).toString val version = resource.get(GovernanceConstant.TASK_RESOURCE_VERSION_STR).toString val fullPath = fileName +// if (workDir.endsWith(seperator)) pathType + workDir + fileName +// else pathType + workDir + seperator + fileName val response = Utils.tryCatch { bmlClient.downloadShareResource(processUser, resourceId, version, fullPath, true) } { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala index bcd423fd21..a3f7bb49e3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala @@ -60,7 +60,7 @@ object ComputationExecutorConf { ).getValue val ENGINE_CONCURRENT_THREAD_NUM = CommonVars( - "wds.linkis.engineconn.concurrent.thread.num", + "linkis.engineconn.concurrent.thread.num", 20, "Maximum thread pool of the concurrent EC" ) @@ -130,4 +130,10 @@ object ComputationExecutorConf { val ENGINE_SEND_LOG_TO_ENTRANCE_LIMIT_LENGTH = CommonVars("linkis.ec.send.log.entrance.limit.length", 2000) + val ENGINE_KERBEROS_AUTO_REFRESH_ENABLED = + CommonVars("linkis.ec.kerberos.auto.refresh.enabled", false).getValue + + val CLOSE_RS_OUTPUT_WHEN_RESET_BY_DEFAULT_ENABLED = + CommonVars("linkis.ec.rs.close.when.reset.enabled", true).getValue + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableRegister.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableRegister.scala index 8b9f95d66d..cc899cece4 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableRegister.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableRegister.scala @@ -88,7 +88,7 @@ object CSTableRegister extends Logging { val csColumns = columns.map { column => val csColumn = new CSColumn csColumn.setName(column.columnName) - csColumn.setType(column.dataType.getTypeName) + csColumn.setType(column.dataType.typeName) csColumn.setComment(column.comment) csColumn } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableResultSetWriter.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableResultSetWriter.scala index 891f37f27e..163d7cf57e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableResultSetWriter.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSTableResultSetWriter.scala @@ -67,7 +67,7 @@ class CSTableResultSetWriter( val csColumns = getMetaData.asInstanceOf[TableMetaData].columns.map { column => val csColumn = new CSColumn csColumn.setName(column.columnName) - csColumn.setType(column.dataType.getTypeName) + csColumn.setType(column.dataType.typeName) csColumn.setComment(column.comment) csColumn } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala index bd6d44e4a6..5a06ca007f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala @@ -21,14 +21,17 @@ import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor +import org.apache.linkis.engineconn.acessible.executor.info.DefaultNodeHealthyInfoManager import org.apache.linkis.engineconn.acessible.executor.listener.event.{ TaskLogUpdateEvent, TaskResponseErrorEvent, TaskStatusChangedEvent } +import org.apache.linkis.engineconn.acessible.executor.utils.AccessibleExecutorUtils.currentEngineIsUnHealthy import org.apache.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant} import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask +import org.apache.linkis.engineconn.computation.executor.exception.HookExecuteException import org.apache.linkis.engineconn.computation.executor.hook.ComputationExecutorHook import org.apache.linkis.engineconn.computation.executor.metrics.ComputationEngineConnMetrics import org.apache.linkis.engineconn.computation.executor.upstream.event.TaskStatusChangedForUpstreamMonitorEvent @@ -40,7 +43,7 @@ import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.paser.CodeParser import org.apache.linkis.governance.common.protocol.task.{EngineConcurrentInfo, RequestTask} import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus +import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus} import org.apache.linkis.manager.label.entity.engine.{ CodeLanguageLabel, EngineType, @@ -60,6 +63,7 @@ import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConverters._ +import DataWorkCloudApplication.getApplicationContext import com.google.common.cache.{Cache, CacheBuilder} abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) @@ -70,6 +74,7 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) private val listenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext() + // private val taskMap: util.Map[String, EngineConnTask] = new ConcurrentHashMap[String, EngineConnTask](8) private val taskCache: Cache[String, EngineConnTask] = CacheBuilder .newBuilder() .expireAfterAccess(EngineConnConf.ENGINE_TASK_EXPIRE_TIME.getValue, TimeUnit.MILLISECONDS) @@ -84,13 +89,17 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) protected val runningTasks: Count = new Count + protected val pendingTasks: Count = new Count + protected val succeedTasks: Count = new Count protected val failedTasks: Count = new Count - private var lastTask: EngineConnTask = _ + protected var lastTask: EngineConnTask = _ - private val MAX_TASK_EXECUTE_NUM = ComputationExecutorConf.ENGINE_MAX_TASK_EXECUTE_NUM.getValue + private val MAX_TASK_EXECUTE_NUM = ComputationExecutorConf.ENGINE_MAX_TASK_EXECUTE_NUM.getValue( + EngineConnObject.getEngineCreationContext.getOptions + ) private val CLOSE_LOCKER = new Object @@ -156,6 +165,8 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) super.close() } + // override def getName: String = ComputationExecutorConf.DEFAULT_COMPUTATION_NAME + protected def ensureOp[A](f: => A): A = if (!isEngineInitialized) { f } else ensureIdle(f) @@ -166,6 +177,11 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) engineConnTask: EngineConnTask, executeResponse: ExecuteResponse ): Unit = { + Utils.tryAndWarn { + ComputationExecutorHook.getComputationExecutorHooks.foreach { hook => + hook.afterExecutorExecute(engineConnTask, executeResponse) + } + } val executorNumber = getSucceedNum + getFailedNum if ( MAX_TASK_EXECUTE_NUM > 0 && runningTasks @@ -174,6 +190,13 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) logger.error(s"Task has reached max execute number $MAX_TASK_EXECUTE_NUM, now tryShutdown. ") ExecutorManager.getInstance.getReportExecutor.tryShutdown() } + + // unhealthy node should try to shutdown + if (runningTasks.getCount() == 0 && currentEngineIsUnHealthy) { + logger.info("no task running and ECNode is unHealthy, now to mark engine to Finished.") + ExecutorManager.getInstance.getReportExecutor.tryShutdown() + } + } def toExecuteTask( @@ -194,7 +217,15 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) hookedCode = hook.beforeExecutorExecute(engineExecutionContext, engineCreationContext, hookedCode) }) - }(e => logger.info("failed to do with hook", e)) + } { e => + e match { + case hookExecuteException: HookExecuteException => + failedTasks.increase() + logger.error("failed to do with hook", e) + return ErrorExecuteResponse("hook execute failed task will be failed", e) + case _ => logger.info("failed to do with hook", e) + } + } if (hookedCode.length > 100) { logger.info(s"hooked after code: ${hookedCode.substring(0, 100)} ....") } else { @@ -237,6 +268,7 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) } else executeLine(engineExecutionContext, code)) { t => ErrorExecuteResponse(ExceptionUtils.getRootCauseMessage(t), t) } + // info(s"Finished to execute task ${engineConnTask.getTaskId}") incomplete ++= code response match { case e: ErrorExecuteResponse => @@ -313,6 +345,7 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) Utils.tryAndWarn(afterExecute(engineConnTask, response)) logger.info(s"Finished to execute task ${engineConnTask.getTaskId}") + // lastTask = null response } { LoggerUtils.removeJobIdMDC() @@ -385,7 +418,7 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) EngineConnObject.getEngineCreationContext.getOptions.asScala.foreach({ case (key, value) => // skip log jobId because it corresponding jobid when the ec created if (!ComputationExecutorConf.PRINT_TASK_PARAMS_SKIP_KEYS.getValue.contains(key)) { - sb.append(s"${key}=${value.toString}\n") + sb.append(s"${key}=${value}\n") } }) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala index f192a45254..9584dd847c 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala @@ -17,9 +17,15 @@ package org.apache.linkis.engineconn.computation.executor.execute +import org.apache.linkis.DataWorkCloudApplication.getApplicationContext +import org.apache.linkis.engineconn.acessible.executor.info.DefaultNodeHealthyInfoManager +import org.apache.linkis.engineconn.acessible.executor.utils.AccessibleExecutorUtils.currentEngineIsUnHealthy +import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask +import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.entity.ConcurrentExecutor -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus +import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus} +import org.apache.linkis.manager.label.entity.entrance.ExecuteOnceLabel import org.apache.linkis.scheduler.executer.ExecuteResponse abstract class ConcurrentComputationExecutor(override val outputPrintLimit: Int = 1000) @@ -42,9 +48,7 @@ abstract class ConcurrentComputationExecutor(override val outputPrintLimit: Int transition(NodeStatus.Busy) } } - logger.info( - s"engineConnTask(${engineConnTask.getTaskId}) running task is ($getRunningTask) status ${getStatus}" - ) + logger.info(s"engineConnTask(${engineConnTask.getTaskId}) running task is ($getRunningTask) ") val response = super.execute(engineConnTask) if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) { EXECUTOR_STATUS_LOCKER.synchronized { @@ -64,10 +68,47 @@ abstract class ConcurrentComputationExecutor(override val outputPrintLimit: Int override def afterExecute( engineConnTask: EngineConnTask, executeResponse: ExecuteResponse - ): Unit = {} + ): Unit = { + // execute once should try to shutdown + if (engineConnTask.getLables.exists(_.isInstanceOf[ExecuteOnceLabel])) { + if (!hasTaskRunning()) { + logger.warn( + s"engineConnTask(${engineConnTask.getTaskId}) is execute once, now to mark engine to Finished" + ) + ExecutorManager.getInstance.getReportExecutor.tryShutdown() + } + } + // unhealthy node should try to shutdown + if (!hasTaskRunning() && currentEngineIsUnHealthy) { + logger.info("no task running and ECNode is unHealthy, now to mark engine to Finished.") + ExecutorManager.getInstance.getReportExecutor.tryShutdown() + } + } override def hasTaskRunning(): Boolean = { getRunningTask > 0 } + override def transition(toStatus: NodeStatus): Unit = { + if (getRunningTask >= getConcurrentLimit && NodeStatus.Unlock == toStatus) { + logger.info( + s"running task($getRunningTask) > concurrent limit:$getConcurrentLimit, can not to mark EC to Unlock" + ) + return + } + super.transition(toStatus) + } + + override def getConcurrentLimit: Int = { + var maxTaskNum = ComputationExecutorConf.ENGINE_CONCURRENT_THREAD_NUM.getValue - 5 + if (maxTaskNum <= 0) { + logger.error( + s"max task num cannot ${maxTaskNum} < 0, should set linkis.engineconn.concurrent.thread.num > 6" + ) + maxTaskNum = 1 + } + logger.info(s"max task num $maxTaskNum") + maxTaskNum + } + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala index 377c32c193..4a13da3c25 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/EngineExecutionContext.scala @@ -21,12 +21,14 @@ import org.apache.linkis.common.io.{FsPath, MetaData, Record} import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetWriter} import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.cs.client.utils.ContextServiceUtils +import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration import org.apache.linkis.engineconn.acessible.executor.listener.event.{ TaskLogUpdateEvent, TaskProgressUpdateEvent, TaskResultCreateEvent, TaskResultSizeCreatedEvent } +import org.apache.linkis.engineconn.acessible.executor.log.LogHelper import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf import org.apache.linkis.engineconn.computation.executor.cs.CSTableResultSetWriter import org.apache.linkis.engineconn.executor.ExecutorExecutionContext @@ -40,7 +42,7 @@ import org.apache.linkis.governance.common.exception.engineconn.EngineConnExecut import org.apache.linkis.protocol.engine.JobProgressInfo import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse} import org.apache.linkis.storage.{LineMetaData, LineRecord} -import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetWriterFactory} +import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} import org.apache.linkis.storage.resultset.table.TableResultSet import org.apache.commons.io.IOUtils @@ -55,8 +57,7 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String private val resultSetFactory = ResultSetFactory.getInstance - private var defaultResultSetWriter - : org.apache.linkis.common.io.resultset.ResultSetWriter[_ <: MetaData, _ <: Record] = _ + private var defaultResultSetWriter: ResultSetWriter[_ <: MetaData, _ <: Record] = _ private var resultSize = 0 @@ -88,12 +89,7 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String * Note: the writer will be closed at the end of the method * @param resultSetWriter */ - def sendResultSet( - resultSetWriter: org.apache.linkis.common.io.resultset.ResultSetWriter[ - _ <: MetaData, - _ <: Record - ] - ): Unit = { + def sendResultSet(resultSetWriter: ResultSetWriter[_ <: MetaData, _ <: Record]): Unit = { logger.info("Start to send res to entrance") val fileName = new File(resultSetWriter.toFSPath.getPath).getName val index = if (fileName.indexOf(".") < 0) fileName.length else fileName.indexOf(".") @@ -148,13 +144,13 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String ): ResultSet[_ <: MetaData, _ <: Record] = resultSetFactory.getResultSetByType(resultSetType) - override protected def getDefaultResultSetByType: String = resultSetFactory.getResultSetType()(0) + override protected def getDefaultResultSetByType: String = resultSetFactory.getResultSetType(0) def newResultSetWriter( resultSet: ResultSet[_ <: MetaData, _ <: Record], resultSetPath: FsPath, alias: String - ): org.apache.linkis.common.io.resultset.ResultSetWriter[_ <: MetaData, _ <: Record] = { + ): ResultSetWriter[_ <: MetaData, _ <: Record] = { // update by 20200402 resultSet match { case result: TableResultSet => @@ -172,7 +168,7 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String csWriter.setProxyUser(executorUser) csWriter } else { - ResultSetWriterFactory.getResultSetWriter( + ResultSetWriter.getResultSetWriter( resultSet, ComputationExecutorConf.ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, @@ -180,7 +176,7 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String ) } case _ => - ResultSetWriterFactory.getResultSetWriter( + ResultSetWriter.getResultSetWriter( resultSet, ComputationExecutorConf.ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, @@ -194,15 +190,20 @@ class EngineExecutionContext(executor: ComputationExecutor, executorUser: String logger.info(log) } else { var taskLog = log + val limitLength = ComputationExecutorConf.ENGINE_SEND_LOG_TO_ENTRANCE_LIMIT_LENGTH.getValue if ( ComputationExecutorConf.ENGINE_SEND_LOG_TO_ENTRANCE_LIMIT_ENABLED.getValue && - log.length > ComputationExecutorConf.ENGINE_SEND_LOG_TO_ENTRANCE_LIMIT_LENGTH.getValue + log.length > limitLength ) { - taskLog = - s"${log.substring(0, ComputationExecutorConf.ENGINE_SEND_LOG_TO_ENTRANCE_LIMIT_LENGTH.getValue)}..." + taskLog = s"${log.substring(0, limitLength)}..." + logger.info("The log is too long and will be intercepted,log limit length : {}", limitLength) + } + if (!AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM.getValue) { + LogHelper.cacheLog(taskLog) + } else { + val listenerBus = getEngineSyncListenerBus + getJobId.foreach(jId => listenerBus.postToAll(TaskLogUpdateEvent(jId, taskLog))) } - val listenerBus = getEngineSyncListenerBus - getJobId.foreach(jId => listenerBus.postToAll(TaskLogUpdateEvent(jId, taskLog))) } override def close(): Unit = { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala index c23e28a3a3..f9164b2cf3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala @@ -19,7 +19,9 @@ package org.apache.linkis.engineconn.computation.executor.hook import org.apache.linkis.common.utils.{ClassUtils, Logging, Utils} import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext +import org.apache.linkis.scheduler.executer.ExecuteResponse import scala.collection.JavaConverters.asScalaSetConverter import scala.collection.mutable.ArrayBuffer @@ -36,6 +38,11 @@ trait ComputationExecutorHook { codeBeforeHook: String ): String = codeBeforeHook + def afterExecutorExecute( + engineConnTask: EngineConnTask, + executeResponse: ExecuteResponse + ): Unit = {} + } object ComputationExecutorHook extends Logging { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/InitSQLHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/InitSQLHook.scala index 78eb007d2c..0cb33cf3d7 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/InitSQLHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/InitSQLHook.scala @@ -33,7 +33,6 @@ import org.apache.commons.io.FileUtils import org.apache.commons.lang3.StringUtils import java.io.File -import java.nio.charset.StandardCharsets import java.util abstract class InitSQLHook extends EngineConnHook with Logging { @@ -92,7 +91,7 @@ abstract class InitSQLHook extends EngineConnHook with Logging { logger.info("read file: " + path) val file = new File(path) if (file.exists()) { - FileUtils.readFileToString(file, StandardCharsets.UTF_8) + FileUtils.readFileToString(file) } else { logger.info("file: [" + path + "] doesn't exist, ignore it.") "" diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala new file mode 100644 index 0000000000..34928d8525 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf +import org.apache.linkis.engineconn.computation.executor.execute.{ + ComputationExecutor, + EngineExecutionContext +} +import org.apache.linkis.engineconn.core.engineconn.EngineConnManager +import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.rpc.Sender +import org.apache.linkis.udf.UDFClientConfiguration +import org.apache.linkis.udf.api.rpc.{RequestPythonModuleProtocol, ResponsePythonModuleProtocol} +import org.apache.linkis.udf.entity.PythonModuleInfoVO + +import org.apache.commons.lang3.StringUtils + +import java.util + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** + * The PythonModuleLoad class is designed to load Python modules into the execution environment + * dynamically. This class is not an extension of UDFLoad, but shares a similar philosophy of + * handling dynamic module loading based on user preferences and system configurations. + */ +abstract class PythonModuleLoad extends Logging { + + /** Abstract properties to be defined by the subclass */ + protected val engineType: String + protected val runType: RunType + + protected def getEngineType(): String = engineType + + protected def constructCode(pythonModuleInfo: PythonModuleInfoVO): String + + private def queryPythonModuleRpc( + userName: String, + engineType: String + ): java.util.List[PythonModuleInfoVO] = { + val infoList = Sender + .getSender(UDFClientConfiguration.UDF_SERVICE_NAME.getValue) + .ask(RequestPythonModuleProtocol(userName, engineType)) + .asInstanceOf[ResponsePythonModuleProtocol] + .getModulesInfo() + infoList + } + + protected def getLoadPythonModuleCode: Array[String] = { + val engineCreationContext = + EngineConnManager.getEngineConnManager.getEngineConn.getEngineCreationContext + val user = engineCreationContext.getUser + + var infoList: util.List[PythonModuleInfoVO] = + Utils.tryAndWarn(queryPythonModuleRpc(user, getEngineType())) + if (infoList == null) { + logger.info("rpc get info is empty.") + infoList = new util.ArrayList[PythonModuleInfoVO]() + } + + // 替换Viewfs + if (IS_VIEW_FS_ENV.getValue) { + infoList.asScala.foreach { info => + val path = info.getPath + logger.info(s"python path: ${path}") + if (path.startsWith("hdfs") || path.startsWith("viewfs")) { + info.setPath(path.replace("hdfs://", "viewfs://")) + } else { + info.setPath("viewfs://" + path) + } + } + } else { + + infoList.asScala.foreach { info => + val path = info.getPath + logger.info(s"hdfs python path: ${path}") + if (!path.startsWith("hdfs")) { + info.setPath("hdfs://" + path) + } + } + } + + logger.info(s"${user} load python modules: ") + infoList.asScala.foreach(l => logger.info(s"module name:${l.getName}, path:${l.getPath}\n")) + + // 创建加载code + val codes: mutable.Buffer[String] = infoList.asScala + .filter { info => StringUtils.isNotEmpty(info.getPath) } + .map(constructCode) + // 打印codes + val str: String = codes.mkString("\n") + logger.info(s"python codes: $str") + codes.toArray + } + + private def executeFunctionCode(codes: Array[String], executor: ComputationExecutor): Unit = { + if (null == codes || null == executor) { + return + } + codes.foreach { code => + logger.info("Submit function registration to engine, code: " + code) + Utils.tryCatch(executor.executeLine(new EngineExecutionContext(executor), code)) { + t: Throwable => + logger.error("Failed to load python module", t) + null + } + } + } + + /** + * Generate and execute the code necessary for loading Python modules. + * + * @param executor + * An object capable of executing code in the current engine context. + */ + protected def loadPythonModules(labels: Array[Label[_]]): Unit = { + + val codes = getLoadPythonModuleCode + logger.info(s"codes length: ${codes.length}") + if (null != codes && codes.nonEmpty) { + val executor = ExecutorManager.getInstance.getExecutorByLabels(labels) + if (executor != null) { + val className = executor.getClass.getName + logger.info(s"executor class: ${className}") + } else { + logger.error(s"Failed to load python, executor is null") + } + + executor match { + case computationExecutor: ComputationExecutor => + executeFunctionCode(codes, computationExecutor) + case _ => + } + } + logger.info(s"Successful to load python, engineType : ${engineType}") + } + +} + +// Note: The actual implementation of methods like `executeFunctionCode` and `construct diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala new file mode 100644 index 0000000000..80eaa888b8 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel + +abstract class PythonModuleLoadEngineConnHook + extends PythonModuleLoad + with EngineConnHook + with Logging { + + override def afterExecutionExecute( + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): Unit = { + Utils.tryAndWarnMsg { + val codeLanguageLabel = new CodeLanguageLabel + codeLanguageLabel.setCodeType(runType.toString) + logger.info(s"engineType: ${engineType}") + val labels = Array[Label[_]](codeLanguageLabel) + loadPythonModules(labels) + }(s"Failed to load Python Modules: ${engineType}") + + } + + override def afterEngineServerStartFailed( + engineCreationContext: EngineCreationContext, + throwable: Throwable + ): Unit = { + logger.error(s"Failed to start Engine Server: ${throwable.getMessage}", throwable) + } + + override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = { + logger.info("Preparing to load Python Module...") + } + + override def beforeExecutionExecute( + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): Unit = { + logger.info(s"Before executing command on load Python Module.") + } + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala new file mode 100644 index 0000000000..0fe554f93d --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.manager.label.entity.engine.RunType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.udf.entity.PythonModuleInfoVO + +/** + * 定义一个用于Spark引擎的Python模块加载与执行挂钩的类 + */ +class PythonSparkEngineHook extends PythonModuleLoadEngineConnHook { + + // 设置engineType属性为"spark"，表示此挂钩适用于Spark数据处理引擎 + override val engineType: String = "spark" + + // 设置runType属性为RunType.PYSPARK，表示此挂钩将执行PySpark类型的代码 + override protected val runType: RunType = RunType.PYSPARK + + // 重写constructCode方法，用于根据Python模块信息构造加载模块的代码 + override protected def constructCode(pythonModuleInfo: PythonModuleInfoVO): String = { + // 使用pythonModuleInfo的path属性，构造SparkContext.addPyFile的命令字符串 + // 这个命令在PySpark环境中将模块文件添加到所有worker上，以便在代码中可以使用 + val path: String = pythonModuleInfo.getPath + val loadCode = s"sc.addPyFile('${path}')" + logger.info(s"pythonLoadCode: ${loadCode}") + loadCode + } + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala index 6f31bd25e8..91af2811a6 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala @@ -42,7 +42,6 @@ import org.apache.commons.io.{FileUtils, IOUtils} import org.apache.commons.lang3.StringUtils import java.io.File -import java.nio.charset.StandardCharsets import scala.collection.JavaConverters.asScalaBufferConverter import scala.collection.mutable.ArrayBuffer @@ -119,7 +118,7 @@ abstract class UDFLoad extends Logging { logger.info("read file: " + path) val file = new File(path) if (file.exists()) { - FileUtils.readFileToString(file, StandardCharsets.UTF_8) + FileUtils.readFileToString(file) } else { logger.info("udf file: [" + path + "] doesn't exist, ignore it.") "" diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UseDatabaseEngineHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UseDatabaseEngineHook.scala index 9ddde065cc..d6887218b7 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UseDatabaseEngineHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UseDatabaseEngineHook.scala @@ -34,7 +34,7 @@ import org.apache.commons.lang3.StringUtils abstract class UseDatabaseEngineHook extends EngineConnHook with Logging { - private val USE_DEFAULT_DB_ENABLE = CommonVars("linkis.bdp.use.default.db.enable", false) + private val USE_DEFAULT_DB_ENABLE = CommonVars("wds.linkis.bdp.use.default.db.enable", true) override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = {} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/executor/ExecuteOnceHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/executor/ExecuteOnceHook.scala index 77616944d2..9469c00685 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/executor/ExecuteOnceHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/executor/ExecuteOnceHook.scala @@ -28,8 +28,11 @@ import org.apache.linkis.engineconn.computation.executor.execute.EngineExecution import org.apache.linkis.engineconn.computation.executor.hook.ComputationExecutorHook import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext +import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.entrance.ExecuteOnceLabel +import java.util + class ExecuteOnceHook extends ComputationExecutorHook with ExecutorLockListener with Logging { private var executeOnce = false @@ -47,6 +50,11 @@ class ExecuteOnceHook extends ComputationExecutorHook with ExecutorLockListener codeBeforeHook: String ): String = { executeOnce = engineExecutionContext.getLabels.exists(_.isInstanceOf[ExecuteOnceLabel]) + val creationLabelList: util.List[Label[_]] = engineCreationContext.getLabels() + if (creationLabelList != null) { + executeOnce = + executeOnce || creationLabelList.toArray().exists(_.isInstanceOf[ExecuteOnceLabel]) + } if (executeOnce && !isRegister) { isRegister = true asyncListenerBusContext.addListener(this) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/rs/RsOutputStream.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/rs/RsOutputStream.scala index 1f8c491ced..55b88f520b 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/rs/RsOutputStream.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/rs/RsOutputStream.scala @@ -19,7 +19,8 @@ package org.apache.linkis.engineconn.computation.executor.rs import org.apache.linkis.common.io.{MetaData, Record} import org.apache.linkis.common.io.resultset.ResultSetWriter -import org.apache.linkis.common.utils.Logging +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext import org.apache.linkis.storage.LineRecord @@ -45,6 +46,9 @@ class RsOutputStream extends OutputStream with Logging { } def reset(engineExecutionContext: EngineExecutionContext): Unit = { + if (ComputationExecutorConf.CLOSE_RS_OUTPUT_WHEN_RESET_BY_DEFAULT_ENABLED) { + Utils.tryQuietly(close()) + } writer = engineExecutionContext.createDefaultResultSetWriter() writer.addMetaData(null) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 9dba95ef66..28df29db31 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -50,6 +50,7 @@ import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.entity.ResourceFetchExecutor import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext import org.apache.linkis.engineconn.executor.listener.event.EngineConnSyncEvent +import org.apache.linkis.engineconn.launch.EngineConnServer import org.apache.linkis.governance.common.constant.ec.ECConstants import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.exception.engineconn.{ @@ -58,11 +59,13 @@ import org.apache.linkis.governance.common.exception.engineconn.{ } import org.apache.linkis.governance.common.protocol.task._ import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} +import org.apache.linkis.hadoop.common.utils.KerberosUtils import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.resource.{ ResponseTaskRunningInfo, ResponseTaskYarnResource } +import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.message.RequestProtocol @@ -106,19 +109,12 @@ class TaskExecutionServiceImpl private lazy val executorManager = ExecutorManager.getInstance private val taskExecutedNum = new AtomicInteger(0) private var lastTask: EngineConnTask = _ - private var lastTaskFuture: Future[_] = _ + private var syncLastTaskThread: Thread = _ private var lastTaskDaemonFuture: Future[_] = _ - // for concurrent executor - private var consumerThread: Thread = _ - private var concurrentTaskQueue: BlockingQueue[EngineConnTask] = _ - @Autowired private var lockService: LockService = _ - private val asyncListenerBusContext = - ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus - private val syncListenerBus = ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnSyncListenerBus @@ -133,18 +129,19 @@ class TaskExecutionServiceImpl "ConcurrentEngineConnThreadPool" ) - private val CONCURRENT_TASK_LOCKER = new Object - private val taskAsyncSubmitExecutor: ExecutionContextExecutorService = Utils.newCachedExecutionContext( ComputationExecutorConf.TASK_ASYNC_MAX_THREAD_SIZE, - "TaskExecution-Thread-" + ComputationEngineConstant.TASK_EXECUTION_THREAD ) @PostConstruct def init(): Unit = { LogHelper.setLogListener(this) syncListenerBus.addListener(this) + if (ComputationExecutorConf.ENGINE_KERBEROS_AUTO_REFRESH_ENABLED) { + KerberosUtils.startKerberosRefreshThread() + } } private def sendToEntrance(task: EngineConnTask, msg: RequestProtocol): Unit = { @@ -154,7 +151,8 @@ class TaskExecutionServiceImpl sender = Sender.getSender(task.getCallbackServiceInstance()) sender.send(msg) } else { - logger.warn("SendtoEntrance error, cannot find entrance instance.") + // todo + logger.debug("SendtoEntrance error, cannot find entrance instance.") } } { t => val errorMsg = s"SendToEntrance error. $msg" + t.getCause @@ -166,6 +164,12 @@ class TaskExecutionServiceImpl } } + /** + * submit to async thread return submit response + * @param requestTask + * @param sender + * @return + */ @Receiver override def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse = Utils.tryFinally { @@ -186,7 +190,7 @@ class TaskExecutionServiceImpl if (!lockService.isLockExist(requestTask.getLock)) { logger.error(s"Lock ${requestTask.getLock} not exist, cannot execute.") return ErrorExecuteResponse( - "Lock not exist", + "Lock not exixt", new EngineConnExecutorErrorException( EngineConnExecutorErrorCode.INVALID_LOCK, "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)." @@ -200,7 +204,11 @@ class TaskExecutionServiceImpl ) } - val taskId: Int = taskExecutedNum.incrementAndGet() + val taskId: String = if (StringUtils.isNotBlank(jobId)) { + jobId + } else { + String.valueOf(taskExecutedNum.incrementAndGet()) + } val retryAble: Boolean = { val retry = requestTask.getProperties.getOrDefault( @@ -215,7 +223,7 @@ class TaskExecutionServiceImpl System.getProperties.put(ComputationExecutorConf.JOB_ID_TO_ENV_KEY, jobId) logger.info(s"Received job with id ${jobId}.") } - val task = new CommonEngineConnTask(String.valueOf(taskId), retryAble) + val task = new CommonEngineConnTask(taskId, retryAble) task.setCode(requestTask.getCode) task.setProperties(requestTask.getProperties) task.data(ComputationEngineConstant.LOCK_TYPE_NAME, requestTask.getLock) @@ -276,19 +284,6 @@ class TaskExecutionServiceImpl } } - private def restExecutorLabels(labels: Array[Label[_]]): Array[Label[_]] = { - var newLabels = labels - ExecutorLabelsRestHook.getExecutorLabelsRestHooks.foreach(hooke => - newLabels = hooke.restExecutorLabels(newLabels) - ) - newLabels - } - - // override def taskStatus(taskID: String): ResponseTaskStatus = { - // val task = taskIdCache.get(taskID) - // ResponseTaskStatus(taskID, task.getStatus.id) - // } - private def submitTask( task: CommonEngineConnTask, computationExecutor: ComputationExecutor @@ -308,100 +303,58 @@ class TaskExecutionServiceImpl task: CommonEngineConnTask, computationExecutor: ComputationExecutor ): ExecuteResponse = { - var response: ExecuteResponse = SubmitResponse(task.getTaskId) - Utils.tryCatch { - computationExecutor.execute(task) - } { t => - logger.error(s"Failed to submit task${task.getTaskId} ", t) - response = ErrorExecuteResponse("Failed to submit task", t) - null - } - response + computationExecutor.execute(task) } private def submitSyncTask( task: CommonEngineConnTask, computationExecutor: ComputationExecutor ): ExecuteResponse = { - val runTask = new Runnable { - override def run(): Unit = Utils.tryAndWarn { - LogHelper.dropAllRemainLogs() - executeTask(task, computationExecutor) - } - } + LogHelper.dropAllRemainLogs() lastTask = task - lastTaskFuture = Utils.defaultScheduler.submit(runTask) - lastTaskDaemonFuture = openDaemonForTask(task, lastTaskFuture, Utils.defaultScheduler) - SubmitResponse(task.getTaskId) + syncLastTaskThread = Thread.currentThread() + lastTaskDaemonFuture = openDaemonForTask(task, Utils.defaultScheduler) + val res = executeTask(task, computationExecutor) + res } private def submitConcurrentTask( task: CommonEngineConnTask, executor: ConcurrentComputationExecutor ): ExecuteResponse = { - if (null == concurrentTaskQueue) CONCURRENT_TASK_LOCKER.synchronized { - if (null == concurrentTaskQueue) { - concurrentTaskQueue = new LinkedBlockingDeque[EngineConnTask]() - } - } - concurrentTaskQueue.put(task) - if (null == consumerThread) CONCURRENT_TASK_LOCKER.synchronized { - if (null == consumerThread) { - consumerThread = new Thread(createConsumerRunnable(executor)) - consumerThread.setDaemon(true) - consumerThread.setName("ConcurrentTaskQueueFifoConsumerThread") - consumerThread.start() - } - } - SubmitResponse(task.getTaskId) - } - - private def createConsumerRunnable(executor: ComputationExecutor): Thread = { - val consumerRunnable = new Runnable { + val concurrentJob = new Runnable { override def run(): Unit = { - var errCount = 0 - val ERR_COUNT_MAX = 20 - while (true) { - Utils.tryCatch { - if (!executor.isBusy && !executor.isClosed) { - val task = concurrentTaskQueue.take() - val concurrentJob = new Runnable { - override def run(): Unit = { - lastTask = task - Utils.tryCatch { - logger.info(s"Start to run task ${task.getTaskId}") - executeTask(task, executor) - } { case t: Throwable => - errCount += 1 - logger.error(s"Execute task ${task.getTaskId} failed :", t) - if (errCount > ERR_COUNT_MAX) { - logger.error( - s"Executor run failed for ${errCount} times over ERROR_COUNT_MAX : ${ERR_COUNT_MAX}, will shutdown." - ) - executor.transition(NodeStatus.ShuttingDown) - } - } - } - } - cachedThreadPool.submit(concurrentJob) - } - Thread.sleep(20) - } { case t: Throwable => - logger.error(s"consumerThread failed :", t) - } + Utils.tryCatch { + val jobId = JobUtils.getJobIdFromMap(task.getProperties) + LoggerUtils.setJobIdMDC(jobId) + logger.info(s"Start to run task ${task.getTaskId}") + executeTask(task, executor) + } { case t: Throwable => + logger.warn("Failed to execute task ", t) + sendToEntrance( + task, + ResponseTaskError(task.getTaskId, ExceptionUtils.getRootCauseMessage(t)) + ) + sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) + LoggerUtils.removeJobIdMDC() + null } } } - new Thread(consumerRunnable) + Utils.tryCatch(cachedThreadPool.submit(concurrentJob)) { case e: Exception => + logger.error(s"Failed to submit task ${task.getTaskId}", e) + throw e + } + SubmitResponse(task.getTaskId) } - private def executeTask(task: EngineConnTask, executor: ComputationExecutor): Unit = + private def executeTask(task: EngineConnTask, executor: ComputationExecutor): ExecuteResponse = Utils.tryFinally { val jobId = JobUtils.getJobIdFromMap(task.getProperties) LoggerUtils.setJobIdMDC(jobId) executor.execute(task) - clearCache(task.getTaskId) } { + clearCache(task.getTaskId) LoggerUtils.removeJobIdMDC() } @@ -414,59 +367,54 @@ class TaskExecutionServiceImpl * scheduler * @return */ - private def openDaemonForTask( - task: EngineConnTask, - taskFuture: Future[_], - scheduler: ExecutorService - ): Future[_] = { + private def openDaemonForTask(task: EngineConnTask, scheduler: ExecutorService): Future[_] = { val sleepInterval = ComputationExecutorConf.ENGINE_PROGRESS_FETCH_INTERVAL.getValue scheduler.submit(new Runnable { override def run(): Unit = { + logger.info(s"start daemon thread ${task.getTaskId}, ${task.getStatus}") Utils.tryQuietly(Thread.sleep(TimeUnit.MILLISECONDS.convert(1, TimeUnit.SECONDS))) - while (null != taskFuture && !taskFuture.isDone) { - if (!ExecutionNodeStatus.isCompleted(task.getStatus)) { - Utils.tryAndWarn { - val progressResponse = Utils.tryCatch(taskProgress(task.getTaskId)) { - case e: Exception => - logger.info("Failed to get progress", e) - null - } - val resourceResponse = Utils.tryCatch(buildResourceMap(task)) { case e: Exception => - logger.info("Failed to get resource", e) + while (!ExecutionNodeStatus.isCompleted(task.getStatus)) { + Utils.tryAndWarn { + val progressResponse = Utils.tryCatch(taskProgress(task.getTaskId)) { + case e: Exception => + logger.info("Failed to get progress", e) null - } - val extraInfoMap = Utils.tryCatch(buildExtraInfoMap(task)) { case e: Exception => - logger.info("Failed to get extra info ", e) - null - } - val resourceMap = - if (null != resourceResponse) resourceResponse.getResourceMap else null - - /** - * It is guaranteed that there must be progress the progress must be greater than or - * equal to 0.1 - */ - val newProgressResponse = if (null == progressResponse) { - ResponseTaskProgress(task.getTaskId, 0.1f, null) - } else if (progressResponse.progress < 0.1f) { - ResponseTaskProgress(task.getTaskId, 0.1f, progressResponse.progressInfo) - } else { - progressResponse - } - val respRunningInfo: ResponseTaskRunningInfo = new ResponseTaskRunningInfo( - newProgressResponse.execId, - newProgressResponse.progress, - newProgressResponse.progressInfo, - resourceMap, - extraInfoMap - ) - sendToEntrance(task, respRunningInfo) } + val resourceResponse = Utils.tryCatch(buildResourceMap(task)) { case e: Exception => + logger.info("Failed to get resource", e) + null + } + val extraInfoMap = Utils.tryCatch(buildExtraInfoMap(task)) { case e: Exception => + logger.info("Failed to get extra info ", e) + null + } + val resourceMap = if (null != resourceResponse) resourceResponse.resourceMap else null + + /** + * It is guaranteed that there must be progress the progress must be greater than or + * equal to 0.1 + */ + val newProgressResponse = if (null == progressResponse) { + ResponseTaskProgress(task.getTaskId, 0.1f, null) + } else if (progressResponse.progress < 0.1f) { + ResponseTaskProgress(task.getTaskId, 0.1f, progressResponse.progressInfo) + } else { + progressResponse + } + val respRunningInfo: ResponseTaskRunningInfo = ResponseTaskRunningInfo( + newProgressResponse.execId, + newProgressResponse.progress, + newProgressResponse.progressInfo, + resourceMap, + extraInfoMap + ) + sendToEntrance(task, respRunningInfo) } Utils.tryQuietly( Thread.sleep(TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS)) ) } + logger.info(s"daemon thread exit ${task.getTaskId}, ${task.getStatus}") } }) } @@ -495,7 +443,7 @@ class TaskExecutionServiceImpl taskYarnResource(task.getTaskId) match { case responseTaskYarnResource: ResponseTaskYarnResource => if ( - responseTaskYarnResource.getResourceMap != null && !responseTaskYarnResource.getResourceMap.isEmpty + responseTaskYarnResource.resourceMap != null && !responseTaskYarnResource.resourceMap.isEmpty ) { responseTaskYarnResource } else { @@ -512,7 +460,7 @@ class TaskExecutionServiceImpl executor match { case executor: ResourceFetchExecutor => val resourceWithApplicationId = executor.FetchResource - new ResponseTaskYarnResource(taskID, resourceWithApplicationId) + ResponseTaskYarnResource(taskID, resourceWithApplicationId) case _ => null } } @@ -553,16 +501,20 @@ class TaskExecutionServiceImpl override def killTask(taskID: String): Unit = { val executor = taskIdCache.getIfPresent(taskID) if (null != executor) { - executor.killTask(taskID) + Utils.tryAndWarn(executor.killTask(taskID)) logger.info(s"TaskId : ${taskID} was killed by user.") } else { logger.error(s"Kill failed, got invalid executor : null for taskId : ${taskID}") } if (null != lastTask && lastTask.getTaskId.equalsIgnoreCase(taskID)) { - if (null != lastTaskFuture && !lastTaskFuture.isDone) { - Utils.tryAndWarn { - lastTaskFuture.cancel(true) - } + if (null != syncLastTaskThread) { + logger.info(s"try to interrupt thread:${taskID}") + Utils.tryAndWarn(syncLastTaskThread.interrupt()) + logger.info(s"thread isInterrupted:${taskID}") + } else { + logger.info(s"skip to force stop thread:${taskID}") + } + if (null != lastTaskDaemonFuture && !lastTaskDaemonFuture.isDone) { Utils.tryAndWarn { // Close the daemon also lastTaskDaemonFuture.cancel(true) @@ -630,7 +582,7 @@ class TaskExecutionServiceImpl if (null != task) { sendToEntrance(task, ResponseTaskLog(logUpdateEvent.taskId, logUpdateEvent.log)) } else { - logger.warn("Task cannot null! logupdateEvent: " + logUpdateEvent.taskId) + logger.error("Task cannot null! logupdateEvent: " + logUpdateEvent.taskId) } } else if (null != lastTask) { val executor = executorManager.getReportExecutor @@ -656,7 +608,6 @@ class TaskExecutionServiceImpl val task = getTaskByTaskId(taskStatusChangedEvent.taskId) if (null != task) { if (ExecutionNodeStatus.isCompleted(taskStatusChangedEvent.toStatus)) { - lastTask = task LogHelper.pushAllRemainLogs() } val toStatus = taskStatusChangedEvent.toStatus @@ -688,9 +639,9 @@ class TaskExecutionServiceImpl val resourceResponse = buildResourceMap(task) val extraInfoMap = buildExtraInfoMap(task) - val resourceMap = if (null != resourceResponse) resourceResponse.getResourceMap else null + val resourceMap = if (null != resourceResponse) resourceResponse.resourceMap else null - val respRunningInfo: ResponseTaskRunningInfo = new ResponseTaskRunningInfo( + val respRunningInfo: ResponseTaskRunningInfo = ResponseTaskRunningInfo( taskProgressUpdateEvent.taskId, taskProgressUpdateEvent.progress, taskProgressUpdateEvent.progressInfo, @@ -732,7 +683,7 @@ class TaskExecutionServiceImpl if (null != executor) { executor.getTaskById(taskId) } else { - logger.warn(s"Executor of taskId : $taskId is not cached.") + logger.error(s"Executor of taskId : $taskId is not cached.") null } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/ECTaskEntranceMonitor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/ECTaskEntranceMonitor.scala index 1f13380c01..c7d78ef9eb 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/ECTaskEntranceMonitor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/ECTaskEntranceMonitor.scala @@ -57,7 +57,7 @@ class ECTaskEntranceMonitor def unregister(taskID: String): Unit = { if (!wrapperMap.containsKey(taskID)) { - logger.error("attempted to unregister non-existing EngineConnTask!! task-id: " + taskID) + logger.warn("attempted to unregister non-existing EngineConnTask!! task-id: " + taskID) } wrapperMap.remove(taskID) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/SingleThreadUpstreamConnectionMonitor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/SingleThreadUpstreamConnectionMonitor.scala index f2b894ef91..3cc1fdfe82 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/SingleThreadUpstreamConnectionMonitor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/SingleThreadUpstreamConnectionMonitor.scala @@ -18,6 +18,8 @@ package org.apache.linkis.engineconn.computation.executor.upstream import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration +import org.apache.linkis.engineconn.acessible.executor.service.ExecutorHeartbeatServiceHolder import org.apache.linkis.engineconn.common.exception.EngineConnException import org.apache.linkis.engineconn.computation.executor.upstream.access.{ ConnectionInfoAccess, @@ -29,6 +31,8 @@ import org.apache.linkis.engineconn.computation.executor.upstream.handler.{ } import org.apache.linkis.engineconn.computation.executor.upstream.wrapper.ConnectionInfoWrapper import org.apache.linkis.engineconn.computation.executor.utlis.ComputationErrorCode +import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.engineconn.executor.entity.ConcurrentExecutor import org.apache.commons.lang3.concurrent.BasicThreadFactory @@ -112,6 +116,23 @@ abstract class SingleThreadUpstreamConnectionMonitor( "requesting connection info: " + util.Arrays .toString(Collections.list(wrapperMap.keys).toArray()) ) + if (AccessibleExecutorConfiguration.ENGINECONN_SUPPORT_PARALLELISM.getValue) { + val executor = ExecutorManager.getInstance.getReportExecutor + executor match { + case concurrentExecutor: ConcurrentExecutor => + if (toBeRequested.size() > (concurrentExecutor.getConcurrentLimit + 20)) { + logger.warn( + s"Executor running task has exceed the limit ${toBeRequested.size()}, executor id ${concurrentExecutor.getId}" + ) + ExecutorHeartbeatServiceHolder + .getDefaultHeartbeatService() + .setSelfUnhealthy( + s"running task has exceed the limit: ${concurrentExecutor.getConcurrentLimit}" + ) + } + case _ => + } + } val infoAccessRequest = generateInfoAccessRequest(toBeRequested) val connectionInfoList = infoAccess.getUpstreamInfo(infoAccessRequest) logger.info( diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/service/ECTaskEntranceMonitorService.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/service/ECTaskEntranceMonitorService.scala index 0acc47260e..44a80cff62 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/service/ECTaskEntranceMonitorService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/service/ECTaskEntranceMonitorService.scala @@ -66,8 +66,7 @@ class ECTaskEntranceMonitorService logger.info("registering new task: " + event.taskId) eCTaskEntranceMonitor.register(event.task, event.executor) } else if ( - fromStatus == ExecutionNodeStatus.Running && - (toStatus == ExecutionNodeStatus.Succeed || toStatus == ExecutionNodeStatus.Failed || toStatus == ExecutionNodeStatus.Cancelled || toStatus == ExecutionNodeStatus.Timeout) + !ExecutionNodeStatus.isCompleted(fromStatus) && ExecutionNodeStatus.isCompleted(toStatus) ) { logger.info("unRegistering task: " + event.taskId) eCTaskEntranceMonitor.unregister(event.task.getTaskId) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/utlis/ComputationEngineConstant.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/utlis/ComputationEngineConstant.scala index 2a03b405cb..4e50fc0cf0 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/utlis/ComputationEngineConstant.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/utlis/ComputationEngineConstant.scala @@ -27,4 +27,6 @@ object ComputationEngineConstant { def CS_HOOK_ORDER: Int = -1 + val TASK_EXECUTION_THREAD = "TaskExecution-Thread-" + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala index 513eac0bd0..a20358b57c 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala @@ -81,13 +81,9 @@ object EngineConnConf { val HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX = CommonVars("wds.linkis.hive.engine.yarn.app.id.parse.regex", "(application_\\d{13}_\\d+)") - val SEATUNNEL_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX = - CommonVars("wds.linkis.seatunnel.engine.yarn.app.id.parse.regex", "(application_\\d{13}_\\d+)") - val JOB_YARN_TASK_URL = CommonVars("linkis.job.task.yarn.url", ""); val JOB_YARN_CLUSTER_TASK_URL = CommonVars("linkis.job.task.yarn.cluster.url", ""); - def getWorkHome: String = System.getenv(ENGINE_CONN_LOCAL_PATH_PWD_KEY.getValue) def getEngineTmpDir: String = System.getenv(ENGINE_CONN_LOCAL_TMP_DIR.getValue) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConstant.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConstant.scala index cdc14592d8..fec2756f9f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConstant.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConstant.scala @@ -27,5 +27,5 @@ object EngineConnConstant { var hiveLogReg = "The url to track the job: http://.*?/proxy/(application_[0-9]+_[0-9]+)/" - val YARN_LOG_URL = "INFO yarn application url:" + val YARN_LOG_URL = "Yarn application url:" } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala index aa69bdae8c..d2247a6d2e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala @@ -17,7 +17,7 @@ package org.apache.linkis.engineconn.core.util -import com.google.gson.{Gson, GsonBuilder, ToNumberPolicy} +import com.google.gson.{GsonBuilder, ToNumberPolicy} object EngineConnUtils { diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/AbstractLogCache.java similarity index 50% rename from linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.java rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/AbstractLogCache.java index f8e01a682c..a1d74fadc6 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/script/compaction/CommonScriptCompaction.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/AbstractLogCache.java @@ -15,33 +15,24 @@ * limitations under the License. */ -package org.apache.linkis.storage.script.compaction; +package org.apache.linkis.engineconn.acessible.executor.log; -import org.apache.linkis.storage.script.Compaction; -import org.apache.linkis.storage.script.Variable; +import org.apache.linkis.common.log.LogUtils; -public abstract class CommonScriptCompaction implements Compaction { +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LogEvent; - @Override - public String compact(Variable variable) { - switch (variable.getSortParent()) { - case "variable": - return prefix() + " " + variable.getKey() + "=" + variable.getValue(); - default: - return prefixConf() - + " " - + variable.getSortParent() - + " " - + variable.getSort() - + " " - + variable.getKey() - + "=" - + variable.getValue(); +public abstract class AbstractLogCache implements LogCache { + protected String generateLog(LogEvent event) { + if (event.getLevel() == Level.INFO) { + return LogUtils.generateInfo(event.getMessage().toString()); + } else if (event.getLevel() == Level.WARN) { + return LogUtils.generateWarn(event.getMessage().toString()); + } else if (event.getLevel() == Level.ERROR) { + return LogUtils.generateERROR(event.getMessage().toString()); + } else if (event.getLevel() == Level.FATAL) { + return LogUtils.generateSystemError(event.getMessage().toString()); } - } - - @Override - public String getAnnotationSymbol() { - return prefix().split("@")[0]; + return ""; } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/MountLogCache.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/MountLogCache.java index e158bdc86b..e80298cef1 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/MountLogCache.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/MountLogCache.java @@ -27,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class MountLogCache implements LogCache { +public class MountLogCache extends AbstractLogCache { private static final Logger logger = LoggerFactory.getLogger(MountLogCache.class); diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/SendAppender.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/SendAppender.java index b27464285d..05976bb2c7 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/SendAppender.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/SendAppender.java @@ -20,16 +20,18 @@ import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration; import org.apache.linkis.engineconn.common.conf.EngineConnConf; import org.apache.linkis.engineconn.common.conf.EngineConnConstant; +import org.apache.linkis.engineconn.common.creation.EngineCreationContext; +import org.apache.linkis.engineconn.core.EngineConnObject; import org.apache.linkis.engineconn.executor.listener.EngineConnSyncListenerBus; import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext; +import org.apache.linkis.manager.label.constant.LabelKeyConstant; +import org.apache.linkis.manager.label.entity.Label; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.appender.AbstractAppender; -import org.apache.logging.log4j.core.config.Property; import org.apache.logging.log4j.core.config.plugins.Plugin; import org.apache.logging.log4j.core.config.plugins.PluginAttribute; import org.apache.logging.log4j.core.config.plugins.PluginElement; @@ -70,9 +72,12 @@ public SendAppender( final Filter filter, final Layout layout, final boolean ignoreExceptions) { - super(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY); + super(name, filter, layout, ignoreExceptions); this.logCache = LogHelper.logCache(); + // SendThread thread = new SendThread(); logger.info("SendAppender init success"); + // TIMER.schedule(thread, 2000, (Integer) + // AccessibleExecutorConfiguration.ENGINECONN_LOG_SEND_TIME_INTERVAL().getValue()); } @Override @@ -96,7 +101,7 @@ public void append(LogEvent event) { } } if (!flag) { - logStr = matchLog(logStr); + // logStr = matchLog(logStr); logCache.cacheLog(logStr); } } else { @@ -120,15 +125,26 @@ public static SendAppender createAppender( return new SendAppender(name, filter, layout, ignoreExceptions); } + /** + * * + * + *

Match the hive log, if it matches the yarn log, print the log and replace it + */ public String matchLog(String logLine) { - String yarnUrl = EngineConnConf.JOB_YARN_TASK_URL().getValue(); - if (StringUtils.isNotBlank(yarnUrl)) { - Matcher hiveMatcher = Pattern.compile(EngineConnConstant.hiveLogReg()).matcher(logLine); - if (hiveMatcher.find()) { - logLine = - hiveMatcher.replaceAll( - EngineConnConstant.YARN_LOG_URL() + yarnUrl + hiveMatcher.group(1)); + Matcher hiveMatcher = Pattern.compile(EngineConnConstant.hiveLogReg()).matcher(logLine); + if (hiveMatcher.find()) { + String yarnUrl = EngineConnConf.JOB_YARN_TASK_URL().getValue(); + EngineCreationContext engineContext = EngineConnObject.getEngineCreationContext(); + if (null != engineContext) { + for (Label label : engineContext.getLabels()) { + if (label.getLabelKey().equals(LabelKeyConstant.YARN_CLUSTER_KEY)) { + yarnUrl = EngineConnConf.JOB_YARN_CLUSTER_TASK_URL().getValue(); + } + } } + logLine = + hiveMatcher.replaceAll( + EngineConnConstant.YARN_LOG_URL() + yarnUrl + hiveMatcher.group(1)); } return logLine; } diff --git a/linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/listener/test/TestContextValue.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/TimeLogCache.java similarity index 68% rename from linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/listener/test/TestContextValue.java rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/TimeLogCache.java index 8c4d4d1447..2ba3efc63d 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/test/java/org/apache/linkis/cs/listener/test/TestContextValue.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/log/TimeLogCache.java @@ -15,30 +15,27 @@ * limitations under the License. */ -package org.apache.linkis.cs.listener.test; +package org.apache.linkis.engineconn.acessible.executor.log; -import org.apache.linkis.cs.common.entity.source.ContextValue; +import java.util.List; -public class TestContextValue implements ContextValue { - private Object value; - - private String keywords; +/** Description: Cache with time as storage unit（以时间作为存储单位的缓存方式） */ +public class TimeLogCache extends AbstractLogCache { + @Override + public void cacheLog(String log) {} @Override - public String getKeywords() { + public List getLog(int num) { return null; } @Override - public void setKeywords(String keywords) {} - - @Override - public Object getValue() { - return this.value; + public List getRemain() { + return null; } @Override - public void setValue(Object value) { - this.value = value; + public int size() { + return 0; } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java deleted file mode 100644 index 66e1c575f0..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.acessible.executor.operator.impl; - -import org.apache.linkis.engineconn.common.exception.EngineConnException; -import org.apache.linkis.engineconn.core.executor.ExecutorManager$; -import org.apache.linkis.engineconn.core.executor.LabelExecutorManager; -import org.apache.linkis.engineconn.executor.entity.Executor; -import org.apache.linkis.engineconn.executor.entity.YarnExecutor; -import org.apache.linkis.governance.common.constant.ec.ECConstants; -import org.apache.linkis.manager.common.operator.Operator; - -import java.util.HashMap; -import java.util.Map; - -public class EngineConnApplicationInfoOperator implements Operator { - - public static final String OPERATOR_NAME = "engineConnYarnApplication"; - - @Override - public String[] getNames() { - return new String[] {OPERATOR_NAME}; - } - - @Override - public Map apply(Map parameters) { - LabelExecutorManager instance = ExecutorManager$.MODULE$.getInstance(); - Executor reportExecutor = instance.getReportExecutor(); - if (reportExecutor instanceof YarnExecutor) { - YarnExecutor yarnExecutor = (YarnExecutor) reportExecutor; - Map result = new HashMap<>(); - result.put(ECConstants.YARN_APPID_NAME_KEY(), yarnExecutor.getApplicationId()); - result.put(ECConstants.YARN_APP_URL_KEY(), yarnExecutor.getApplicationURL()); - result.put(ECConstants.QUEUE(), yarnExecutor.getQueue()); - result.put(ECConstants.YARN_MODE_KEY(), yarnExecutor.getYarnMode()); - return result; - } else { - throw new EngineConnException( - 20301, "EngineConn is not a yarn application, cannot fetch applicaiton info."); - } - } -} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala index 6e6b8b5986..e99f5f21c1 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala @@ -242,7 +242,7 @@ class AccessibleEngineConnExecution extends EngineConnExecution with Logging { case resourceExecutor: ResourceExecutor => ManagerService.getManagerService .reportUsedResource( - new ResourceUsedProtocol( + ResourceUsedProtocol( Sender.getThisServiceInstance, resourceExecutor.getCurrentNodeResource(), engineCreationContext.getTicketId diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/info/NodeHealthyInfoManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/info/NodeHealthyInfoManager.scala index 98f88508e8..6b96b6d4ed 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/info/NodeHealthyInfoManager.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/info/NodeHealthyInfoManager.scala @@ -48,7 +48,7 @@ class DefaultNodeHealthyInfoManager extends NodeHealthyInfoManager with Logging val nodeHealthyInfo = new NodeHealthyInfo nodeHealthyInfo.setMsg("") - /** If it is actively set by the manager, then the manager setting shall prevail */ + /** 如果是manager主动设置的，则以manager设置的为准 */ val newHealthy: NodeHealthy = if (this.setByManager) { this.healthy } else { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/log/LogHelper.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/log/LogHelper.scala index 260e675a1d..a7169697ae 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/log/LogHelper.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/log/LogHelper.scala @@ -44,10 +44,17 @@ object LogHelper extends Logging { def setLogListener(logListener: LogListener): Unit = this.logListener = logListener + def cacheLog(log: String): Unit = { + logCache.cacheLog(log) + } + def pushAllRemainLogs(): Unit = { + // logger.info(s"start to push all remain logs") Thread.sleep(30) + // logCache.synchronized{ if (logListener == null) { logger.warn("logListener is null, can not push remain logs") + // return } else { var logs: util.List[String] = null diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala new file mode 100644 index 0000000000..c7635615e0 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.acessible.executor.operator.impl + +import org.apache.linkis.engineconn.acessible.executor.service.OperateService +import org.apache.linkis.engineconn.common.exception.EngineConnException +import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.engineconn.executor.entity.YarnExecutor +import org.apache.linkis.governance.common.constant.ec.ECConstants._ +import org.apache.linkis.manager.common.operator.Operator + +class EngineConnApplicationInfoOperator extends Operator { + + override def getNames: Array[String] = Array(EngineConnApplicationInfoOperator.OPERATOR_NAME) + + override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { + ExecutorManager.getInstance.getReportExecutor match { + case yarnExecutor: YarnExecutor => + Map( + YARN_APPID_NAME_KEY -> yarnExecutor.getApplicationId, + YARN_APP_URL_KEY -> yarnExecutor.getApplicationURL, + QUEUE -> yarnExecutor.getQueue, + YARN_MODE_KEY -> yarnExecutor.getYarnMode + ) + case _ => + throw EngineConnException( + 20301, + "EngineConn is not a yarn application, cannot fetch applicaiton info." + ) + } + } + +} + +object EngineConnApplicationInfoOperator { + val OPERATOR_NAME = "engineConnYarnApplication" +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala index 7d4dc3b08a..fe3d731b7d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala @@ -57,8 +57,7 @@ class DefaultManagerService extends ManagerService with Logging { logger.info("engineType labels is empty, Not reported") return } - val labelReportRequest = - new LabelReportRequest(reportLabel.asJava, Sender.getThisServiceInstance) + val labelReportRequest = LabelReportRequest(reportLabel.asJava, Sender.getThisServiceInstance) getManagerSender.send(labelReportRequest) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala index 20399711bd..c0ef50636d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala @@ -30,8 +30,6 @@ import org.apache.commons.lang3.exception.ExceptionUtils import org.springframework.stereotype.Service -import java.util - import scala.collection.JavaConverters.mapAsScalaMapConverter @Service @@ -42,36 +40,25 @@ class DefaultOperateService extends OperateService with Logging { engineOperateRequest: EngineOperateRequest ): EngineOperateResponse = { var response: EngineOperateResponse = null - val parameters = { - val map = new util.HashMap[String, Object]() - engineOperateRequest.getParameters.asScala.foreach(entry => map.put(entry._1, entry._2)) - map - } - val operator = Utils.tryCatch(OperatorFactory.apply().getOperatorRequest(parameters)) { t => - logger.error(s"Get operator failed, parameters is ${engineOperateRequest.getParameters}.", t) - response = new EngineOperateResponse( - new util.HashMap[String, Object](), - true, - ExceptionUtils.getRootCauseMessage(t) - ) + + val parameters = engineOperateRequest.parameters.asScala.toMap + val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)) { t => + logger.error(s"Get operator failed, parameters is ${engineOperateRequest.parameters}.", t) + response = EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) doPostHook(engineOperateRequest, response) return response } logger.info( - s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${engineOperateRequest.getParameters}." + s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${engineOperateRequest.parameters}." ) val result = Utils.tryCatch(operator(parameters)) { t => logger.error(s"Execute ${operator.getClass.getSimpleName} failed.", t) - response = new EngineOperateResponse( - new util.HashMap[String, Object](), - true, - ExceptionUtils.getRootCauseMessage(t) - ) + response = EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) doPostHook(engineOperateRequest, response) return response } logger.info(s"End to execute operator ${operator.getClass.getSimpleName}.") - response = new EngineOperateResponse(result) + response = EngineOperateResponse(result) doPostHook(engineOperateRequest, response) response } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala index b5bbc26f92..026234e938 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala @@ -81,12 +81,12 @@ class EngineConnTimedLockService extends LockService with Logging { @throws[EngineConnExecutorErrorException] override def tryLock(requestEngineLock: RequestEngineLock): Option[String] = synchronized { if (null != engineConnLock && engineConnLock.isAcquired()) return None - this.lockType = requestEngineLock.getLockType + this.lockType = requestEngineLock.lockType lockType match { case EngineLockType.Always => timedLock(-1) case EngineLockType.Timed => - timedLock(requestEngineLock.getTimeout) + timedLock(requestEngineLock.timeout) case o: Any => logger.error("Invalid lockType : " + BDPJettyServerHelper.gson.toJson(o)) return Some(null) @@ -172,11 +172,11 @@ class EngineConnTimedLockService extends LockService with Logging { @Receiver override def requestUnLock(requestEngineUnlock: RequestEngineUnlock): ResponseEngineUnlock = { - if (StringUtils.isBlank(requestEngineUnlock.getLock)) { + if (StringUtils.isBlank(requestEngineUnlock.lock)) { logger.error("Invalid requestEngineUnlock: ") - new ResponseEngineUnlock(false) + ResponseEngineUnlock(false) } else { - new ResponseEngineUnlock(unlock(requestEngineUnlock.getLock)) + ResponseEngineUnlock(unlock(requestEngineUnlock.lock)) } } @@ -221,7 +221,7 @@ class EngineConnConcurrentLockService extends LockService { @Receiver override def requestUnLock(requestEngineUnlock: RequestEngineUnlock): ResponseEngineUnlock = - new ResponseEngineUnlock(true) + ResponseEngineUnlock(true) override def onAddLock(addLockEvent: ExecutorLockEvent): Unit = {} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala index 28e4720953..0860076e40 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala @@ -49,15 +49,14 @@ trait LockService extends ExecutorLockListener with Logging { // Engine can be locked if (!StringUtils.isBlank(lockStr)) { // lock success - response = - new ResponseEngineLock(true, lockStr, s"Lock for ${requestEngineLock.getTimeout} ms") + response = ResponseEngineLock(true, lockStr, s"Lock for ${requestEngineLock.timeout} ms") } else { // lock failed - response = new ResponseEngineLock(false, lockStr, "lock str is blank") + response = ResponseEngineLock(false, lockStr, "lock str is blank") } case None => // Engine is busy - response = new ResponseEngineLock(false, null, "Engine is busy.") + response = ResponseEngineLock(false, null, "Engine is busy.") } logger.info( "RequestLock : " + BDPJettyServerHelper.gson.toJson( diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/utils/AccessableExecutorUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/utils/AccessableExecutorUtils.scala index 27dd91cebe..9b4a3ebc28 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/utils/AccessableExecutorUtils.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/utils/AccessableExecutorUtils.scala @@ -26,6 +26,7 @@ object AccessibleExecutorUtils { val manager: DefaultNodeHealthyInfoManager = getApplicationContext.getBean(classOf[DefaultNodeHealthyInfoManager]) + /** 当前引擎是否不健康 不健康返回 true */ def currentEngineIsUnHealthy(): Boolean = { manager != null && manager.getNodeHealthy() == NodeHealthy.UnHealthy } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala index 07cfa51d0a..dfe1137084 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala @@ -29,10 +29,13 @@ import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.core.hook.ShutdownHook import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback +import org.apache.linkis.manager.label.constant.LabelValueConstant +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender import org.apache.linkis.server.conf.ServerConfiguration @@ -60,7 +63,15 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { val newMap = map.++(parser.getSpringConfMap) newMap.put("spring.mvc.servlet.path", ServerConfiguration.BDP_SERVER_RESTFUL_URI.getValue) DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(newMap.toMap)) + val context = EngineConnObject.getEngineCreationContext + val label = LabelUtil.getEngingeConnRuntimeModeLabel(context.getLabels()) + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) { + logger.info("cluster mode call back will be invoke in beforeExecutionExecute") + } else { + val engineConnPidCallBack = new EngineConnIdentifierCallback() + Utils.tryAndError(engineConnPidCallBack.callback()) + } logger.info("<--------------------SpringBoot App init succeed-------------------->") } @@ -68,8 +79,14 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { engineCreationContext: EngineCreationContext, engineConn: EngineConn ): Unit = { - val engineConnIdentifierCallback = new EngineConnIdentifierCallback() - Utils.tryAndError(engineConnIdentifierCallback.callback()) + val context = EngineConnObject.getEngineCreationContext + + val label = LabelUtil.getEngingeConnRuntimeModeLabel(context.getLabels()) + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) { + logger.info("cluster mode call back be invoke") + val engineConnPidCallBack = new EngineConnIdentifierCallback() + Utils.tryAndError(engineConnPidCallBack.callback()) + } } override def afterExecutionExecute( @@ -85,7 +102,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { val prefixMsg = Sender.getThisServiceInstance + s": log dir: ${EngineConnConf.getLogDir}," Utils.tryAndError( engineConnAfterStartCallback.callback( - new EngineConnStatusCallback( + EngineConnStatusCallback( Sender.getThisServiceInstance, engineCreationContext.getTicketId, NodeStatus.Failed, @@ -125,7 +142,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { val engineConnAfterStartCallback = new EngineConnAfterStartCallback Utils.tryAndError( engineConnAfterStartCallback.callback( - new EngineConnStatusCallback( + EngineConnStatusCallback( Sender.getThisServiceInstance, engineCreationContext.getTicketId, getNodeStatusOfStartSuccess(engineCreationContext, engineConn), diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala index efd74e9077..d1eb83d391 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala @@ -35,7 +35,7 @@ abstract class AbstractEngineConnStartUpCallback() extends EngineConnCallback wi def callback(protocol: RequestProtocol): Unit = { protocol match { case protocol: EngineConnStatusCallback => - if (protocol.getStatus().equals(NodeStatus.Failed)) { + if (protocol.status.equals(NodeStatus.Failed)) { logger.error(s"EngineConn Start Failed protocol will send to LM: ${protocol}") } else { logger.info(s"protocol will send to lm: ${protocol}") diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnIdentifierCallback.scala similarity index 99% rename from linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnIdentifierCallback.scala index 71f71f1999..8b9e3ad36e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnIdentifierCallback.scala @@ -30,6 +30,7 @@ import java.lang.management.ManagementFactory class EngineConnIdentifierCallback extends AbstractEngineConnStartUpCallback { override def callback(): Unit = { + var identifier = ManagementFactory.getRuntimeMXBean.getName.split("@")(0) val instance = Sender.getThisServiceInstance val context = EngineConnObject.getEngineCreationContext diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnTimedCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnTimedCallback.scala new file mode 100644 index 0000000000..f323bf2b2f --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnTimedCallback.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.callback.service + +trait EngineConnTimedCallback extends EngineConnCallback {} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/ExecutorExecutionContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/ExecutorExecutionContext.scala index a7675287e0..7f70e21e21 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/ExecutorExecutionContext.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/ExecutorExecutionContext.scala @@ -20,11 +20,11 @@ package org.apache.linkis.engineconn.executor import org.apache.linkis.common.io.{FsPath, MetaData, Record} import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetWriter} import org.apache.linkis.common.utils.Utils -import org.apache.linkis.governance.common.conf.GovernanceCommonConf +import org.apache.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration +import org.apache.linkis.governance.common.utils.GovernanceUtils import org.apache.linkis.manager.label.entity.Label import org.apache.commons.lang3.StringUtils -import org.apache.commons.lang3.time.DateFormatUtils import java.util.concurrent.atomic.AtomicInteger @@ -60,9 +60,12 @@ trait ExecutorExecutionContext { def setLabels(labels: Array[Label[_]]): Unit = this.labels = labels protected def getDefaultStorePath: String = { - val path = GovernanceCommonConf.RESULT_SET_STORE_PATH.getValue - val pathPrefix = (if (path.endsWith("/")) path else path + "/") + Utils.getJvmUser + "/" + - DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd") + "/" + val path = if (EngineConnExecutorConfiguration.LINKIS_RES_DEFAULT_ENABLED) { + GovernanceUtils.getResultParentPath(GovernanceUtils.LINKIS_DEFAULT_RES_CREATOR) + } else { + "hdfs:///apps-data/" + Utils.getJvmUser + } + val pathPrefix = (if (path.endsWith("/")) path else path + "/") + Utils.getJvmUser + "/" getJobId.map(pathPrefix + _ + "/" + System.nanoTime).getOrElse(pathPrefix + System.nanoTime) } @@ -81,11 +84,11 @@ trait ExecutorExecutionContext { protected def getDefaultResultSetByType: String def createDefaultResultSetWriter(): ResultSetWriter[_ <: MetaData, _ <: Record] = { - createResultSetWriter(getResultSetByType(getDefaultResultSetByType)) // todo check + createResultSetWriter(getResultSetByType(getDefaultResultSetByType)) } def createDefaultResultSetWriter(alias: String): ResultSetWriter[_ <: MetaData, _ <: Record] = - createResultSetWriter(getResultSetByType(getDefaultResultSetByType), alias) // todo check + createResultSetWriter(getResultSetByType(getDefaultResultSetByType), alias) def createResultSetWriter(resultSetType: String): ResultSetWriter[_ <: MetaData, _ <: Record] = createResultSetWriter(getResultSetByType(resultSetType), null) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/conf/EngineConnExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/conf/EngineConnExecutorConfiguration.scala index 813022ceb8..f847b9c34f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/conf/EngineConnExecutorConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/conf/EngineConnExecutorConfiguration.scala @@ -66,4 +66,7 @@ object EngineConnExecutorConfiguration { val DEFAULT_EXECUTOR_NAME = CommonVars("wds.linkis.engineconn.executor.default.name", "ComputationExecutor") + val LINKIS_RES_DEFAULT_ENABLED = + CommonVars("wds.linkis.engineconn.res.default.enabled", true).getValue + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java index 034022f169..24c8b904cd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java @@ -30,11 +30,9 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode { CANNOT_DEFAULT_EF(20000, "Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)"), ETL_NOT_EXISTS(20000, "EngineTypeLabel does not exist(EngineTypeLabel 不存在)"), UCL_NOT_EXISTS(20000, "UserCreatorLabel does not exist(UserCreatorLabel 不存在)"), - CANNOT_HOME_PATH_EC( - 20001, "Cannot find the home path of engineConn at: {0}(找不到 engineConn 的 home 路径，该路径为：{0})"), + CANNOT_HOME_PATH_EC(20001, "Cannot find the home path of engineConn(找不到 engineConn 的 home 路径)"), CANNOT_HOME_PATH_DIST( - 20001, - "Could not find the home path for engineconn dist at: {0}(找不到 engineconn dist 的 home 路径，该路径为：{0})"), + 20001, "Cannot find the home path:{0} of engineconn dist(找不到 engineconn dist 的 home 路径)"), DIST_IS_EMPTY( 20001, "The dist of EngineConn is empty,engineConnType is:{0}(EngineConn 的 dist 为空,engineConnType为：{0})"), diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorCode.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorCode.scala new file mode 100644 index 0000000000..5f1537525c --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorCode.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.common.exception + +object EngineConnPluginErrorCode { + + def INVALID_RUNTYPE: Int = 70101 + + def INVALID_LABELS: Int = 70102 + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorException.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorException.scala index d7ab70908a..41cf435459 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorException.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/exception/EngineConnPluginErrorException.scala @@ -17,9 +17,6 @@ package org.apache.linkis.manager.engineplugin.common.exception -import org.apache.linkis.common.exception.{ErrorException, ExceptionLevel, LinkisRuntimeException} +import org.apache.linkis.common.exception.ErrorException -class EngineConnPluginErrorException(code: Int, msg: String) - extends LinkisRuntimeException(code, msg) { - override def getLevel: ExceptionLevel = ExceptionLevel.ERROR -} +class EngineConnPluginErrorException(code: Int, msg: String) extends ErrorException(code, msg) {} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala index aada8caedc..02565a394b 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/resource/UserNodeResource.scala @@ -34,6 +34,9 @@ class UserNodeResource extends NodeResource { private var leftResource: Resource = _ private var createTime: Date = _ private var updateTime: Date = _ + private var maxApps: Int = _ + private var numPendingApps: Int = _ + private var numActiveApps: Int = _ def getUser: String = user @@ -87,4 +90,23 @@ class UserNodeResource extends NodeResource { override def getId: Integer = id override def setId(id: Integer): Unit = this.id = id + + override def getMaxApps: Integer = maxApps + + override def setMaxApps(maxApps: Integer): Unit = { + this.maxApps = maxApps + } + + override def getNumPendingApps: Integer = numPendingApps + + override def setNumPendingApps(numPendingApps: Integer): Unit = { + this.numPendingApps = numPendingApps + } + + override def getNumActiveApps: Integer = numActiveApps + + override def setNumActiveApps(numActiveApps: Integer): Unit = { + this.numActiveApps = numActiveApps + } + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java index 69f15ea864..86af74d5c8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java @@ -66,7 +66,7 @@ public static void generateLogPath(JobRequest jobRequest, Map pa String creator = LabelUtil.getUserCreator(jobRequest.getLabels())._2; String umUser = jobRequest.getExecuteUser(); FsPath lopPrefixPath = new FsPath(logPathPrefix); - if (StorageUtils.HDFS.equals(lopPrefixPath.getFsType())) { + if (StorageUtils.HDFS().equals(lopPrefixPath.getFsType())) { String commonLogPath = logPathPrefix + "/" + "log" + "/" + dateString + "/" + creator; logPath = commonLogPath + "/" + umUser + "/" + jobRequest.getId() + ".log"; CommonLogPathUtils.buildCommonPath(commonLogPath); diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index af7134c5b6..57d2ed04d0 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -515,7 +515,7 @@ private void buildYarnResource( metricsVo.put(TaskConstant.JOB_YARNRESOURCE, resoureList); Optional cores = resourceMap.values().stream() - .map(resource -> resource.getQueueCores()) + .map(resource -> resource.queueCores()) .reduce((x, y) -> x + y); Optional memory = resourceMap.values().stream() diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml index 36076024f2..d6dee78ed2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml @@ -129,6 +129,12 @@ kubernetes-model-core ${kubernetes-client.version} + + + org.apache.linkis + linkis-ps-common-lock + ${project.version} + diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java deleted file mode 100644 index 59d9959431..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.loader; - -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.conf.Configuration; - -public class EngineConnPluginLoaderConf { - - public static final CommonVars ENGINE_PLUGIN_LOADER_DEFAULT_USER = - CommonVars.apply("wds.linkis.engineconn.plugin.loader.defaultUser", "hadoop"); - - public static final CommonVars ENGINE_PLUGIN_STORE_PATH = - CommonVars.apply( - "wds.linkis.engineconn.plugin.loader.store.path", - CommonVars.apply( - "ENGINE_CONN_HOME", - Configuration.getLinkisHome() + "/lib/linkis-engineconn-plugins") - .getValue()); - - public static final CommonVars ENGINE_PLUGIN_PROPERTIES_NAME = - CommonVars.apply("wds.linkis.engineconn.plugin.loader.properties.name", "plugins.properties"); - - public static final CommonVars ENGINE_PLUGIN_LOADER_CACHE_REFRESH_INTERVAL = - CommonVars.apply("wds.linkis.engineconn.plugin.loader.cache.refresh-interval", "300"); - - public static final CommonVars DOWNLOAD_TEMP_DIR_PREFIX = - CommonVars.apply("wds.linkis.engineconn.plugin.loader.download.tmpdir.prefix", ".BML_TMP_"); -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java index 059f984fa4..ef72664460 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java @@ -59,7 +59,8 @@ public CacheablesEngineConnPluginLoader() { @Override public long interval() { return Long.parseLong( - EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_CACHE_REFRESH_INTERVAL.getValue()); + EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_CACHE_REFRESH_INTERVAL() + .getValue()); } @Override diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java index e3ae5ccde1..0e54ed8c4e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java @@ -17,12 +17,13 @@ package org.apache.linkis.engineplugin.loader.loaders; +import org.apache.linkis.common.exception.ErrorException; import org.apache.linkis.engineplugin.loader.EngineConnPluginLoaderConf; import org.apache.linkis.engineplugin.loader.classloader.EngineConnPluginClassLoader; import org.apache.linkis.engineplugin.loader.loaders.resource.LocalEngineConnPluginResourceLoader; import org.apache.linkis.engineplugin.loader.loaders.resource.PluginResource; import org.apache.linkis.engineplugin.loader.utils.EngineConnPluginUtils; -import org.apache.linkis.manager.am.exception.AMErrorException; +import org.apache.linkis.engineplugin.loader.utils.ExceptionHelper; import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin; import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginLoadException; import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginNotFoundException; @@ -57,16 +58,17 @@ public class DefaultEngineConnPluginLoader extends CacheablesEngineConnPluginLoa private static final String PLUGIN_DIR = "plugin"; - public DefaultEngineConnPluginLoader() { + public DefaultEngineConnPluginLoader() throws ErrorException { // Check store path (is necessary) - String storePath = EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH.getValue(); + String storePath = EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH().getValue(); LOG.info("DefaultEngineConnPluginLoader, storePath:" + storePath); if (StringUtils.isBlank(storePath)) { - throw new AMErrorException( + ExceptionHelper.dealErrorException( 70061, "You should defined [" - + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH.key() - + "] in properties file"); + + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH().key() + + "] in properties file", + null); } // The path can be uri try { @@ -78,14 +80,17 @@ public DefaultEngineConnPluginLoader() { } catch (URISyntaxException e) { // Ignore } catch (IllegalArgumentException e) { - throw new AMErrorException( + ExceptionHelper.dealErrorException( 70061, - "You should defined [" - + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH.key() - + "] in properties file"); + "The value:[" + + storePath + + "] of [" + + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH().key() + + "] is incorrect", + e); } this.rootStorePath = storePath; - this.pluginPropsName = EngineConnPluginLoaderConf.ENGINE_PLUGIN_PROPERTIES_NAME.getValue(); + this.pluginPropsName = EngineConnPluginLoaderConf.ENGINE_PLUGIN_PROPERTIES_NAME().getValue(); // Prepare inner loaders // resourceLoaders.add(new BmlEngineConnPluginResourceLoader()); resourceLoaders.add(new LocalEngineConnPluginResourceLoader()); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java index 23607a7063..8b8a071480 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java @@ -58,13 +58,13 @@ public class BmlEngineConnPluginResourceLoader implements EngineConnPluginsResou private String downloadTmpDir; public BmlEngineConnPluginResourceLoader() { - this(EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_DEFAULT_USER.getValue(), null); + this(EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_DEFAULT_USER().getValue(), null); } public BmlEngineConnPluginResourceLoader(String clientUser, Map clientProps) { this.clientUser = clientUser; this.bmlClient = BmlClientFactory.createBmlClient(clientUser, clientProps); - this.downloadTmpDir = EngineConnPluginLoaderConf.DOWNLOAD_TEMP_DIR_PREFIX.getValue(); + this.downloadTmpDir = EngineConnPluginLoaderConf.DOWNLOAD_TEMP_DIR_PREFIX().getValue(); } @Override diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java index 5e71dadc11..2bfcd00aca 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java @@ -93,7 +93,7 @@ private static List getJarsUrlsOfPathRecurse(String path, List classPa parentFile.listFiles( (file) -> { String name = file.getName(); - return !file.isHidden() + return !name.startsWith(".") && (file.isDirectory() || name.endsWith(JAR_SUF_NAME) || name.endsWith(CLASS_SUF_NAME)); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java deleted file mode 100644 index 395471e78b..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.conf; - -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.conf.Configuration; - -public class EngineConnPluginConfiguration { - - public static final CommonVars ENGINE_CONN_HOME = - CommonVars.apply( - "wds.linkis.engineconn.home", - CommonVars.apply( - "ENGINE_CONN_HOME", - Configuration.getLinkisHome() + "/lib/linkis-engineconn-plugins") - .getValue()); - - public static final CommonVars ENGINE_CONN_DIST_LOAD_ENABLE = - CommonVars.apply("wds.linkis.engineconn.dist.load.enable", true); - - public static final CommonVars ENABLED_BML_UPLOAD_FAILED_EXIT = - CommonVars.apply("wds.linkis.engineconn.bml.upload.failed.enable", true); - - // for third party eg appconn/datax, if all update, can set to false then to remove - public static final CommonVars EC_BML_VERSION_MAY_WITH_PREFIX_V = - CommonVars.apply("linkis.engineconn.bml.version.may.with.prefix", true); -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java deleted file mode 100644 index 280c35d660..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.loader; - -import org.apache.linkis.engineplugin.loader.loaders.DefaultEngineConnPluginLoader; - -public class EngineConnPluginsLoaderFactory { - - private static final org.apache.linkis.engineplugin.loader.loaders.EngineConnPluginsLoader - engineConnPluginsLoader = new DefaultEngineConnPluginLoader(); - - public static org.apache.linkis.engineplugin.loader.loaders.EngineConnPluginsLoader - getEngineConnPluginsLoader() { - return engineConnPluginsLoader; - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java deleted file mode 100644 index 2c5d743205..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.localize; - -import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration; -import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; - -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.nio.file.Paths; -import java.text.MessageFormat; -import java.util.Arrays; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.*; - -public abstract class AbstractEngineConnBmlResourceGenerator - implements EngineConnBmlResourceGenerator { - - private static final Logger logger = - LoggerFactory.getLogger(AbstractEngineConnBmlResourceGenerator.class); - - public AbstractEngineConnBmlResourceGenerator() { - if (!new File(getEngineConnsHome()).exists()) { - throw new EngineConnPluginErrorException( - CANNOT_HOME_PATH_EC.getErrorCode(), - MessageFormat.format(CANNOT_HOME_PATH_EC.getErrorDesc(), getEngineConnsHome())); - } - } - - public String getEngineConnsHome() { - return EngineConnPluginConfiguration.ENGINE_CONN_HOME.getValue(); - } - - protected String getEngineConnDistHome(EngineTypeLabel engineConnTypeLabel) { - return getEngineConnDistHome( - engineConnTypeLabel.getEngineType(), engineConnTypeLabel.getVersion()); - } - - protected String getEngineConnDistHome(String engineConnType, String version) { - String engineConnDistHome = - Paths.get(getEngineConnsHome(), engineConnType, "dist").toFile().getPath(); - checkEngineConnDistHome(engineConnDistHome); - if (StringUtils.isBlank(version) - || EngineConnBmlResourceGenerator.NO_VERSION_MARK.equals(version)) { - return engineConnDistHome; - } - String engineConnPackageHome = Paths.get(engineConnDistHome, version).toFile().getPath(); - logger.info("getEngineConnDistHome, engineConnPackageHome path:" + engineConnPackageHome); - File engineConnPackageHomeFile = new File(engineConnPackageHome); - if (!engineConnPackageHomeFile.exists()) { - if (!version.startsWith("v") - && (boolean) EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue()) { - String versionOld = "v" + version; - String engineConnPackageHomeOld = - Paths.get(engineConnDistHome, versionOld).toFile().getPath(); - logger.info( - "try to getEngineConnDistHome with prefix v, engineConnPackageHome path:" - + engineConnPackageHomeOld); - File engineConnPackageHomeFileOld = new File(engineConnPackageHomeOld); - if (!engineConnPackageHomeFileOld.exists()) { - throw new EngineConnPluginErrorException( - ENGINE_VERSION_NOT_FOUND.getErrorCode(), - MessageFormat.format( - ENGINE_VERSION_NOT_FOUND.getErrorDesc(), version, engineConnType)); - } else { - return engineConnPackageHomeOld; - } - } else { - throw new EngineConnPluginErrorException( - ENGINE_VERSION_NOT_FOUND.getErrorCode(), - MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc(), version, engineConnType)); - } - } else { - return engineConnPackageHome; - } - } - - private void checkEngineConnDistHome(String engineConnPackageHomePath) { - File engineConnPackageHomeFile = new File(engineConnPackageHomePath); - checkEngineConnDistHome(engineConnPackageHomeFile); - } - - private void checkEngineConnDistHome(File engineConnPackageHome) { - if (!engineConnPackageHome.exists()) { - throw new EngineConnPluginErrorException( - CANNOT_HOME_PATH_DIST.getErrorCode(), - MessageFormat.format( - CANNOT_HOME_PATH_DIST.getErrorDesc(), engineConnPackageHome.getPath())); - } - } - - protected String[] getEngineConnDistHomeList(String engineConnType) { - String engineConnDistHome = - Paths.get(getEngineConnsHome(), engineConnType, "dist").toFile().getPath(); - File engineConnDistHomeFile = new File(engineConnDistHome); - checkEngineConnDistHome(engineConnDistHomeFile); - File[] children = engineConnDistHomeFile.listFiles(); - if (children.length == 0) { - throw new EngineConnPluginErrorException( - DIST_IS_EMPTY.getErrorCode(), - MessageFormat.format(DIST_IS_EMPTY.getErrorDesc(), engineConnType)); - } else { - return Arrays.stream(children).map(File::getPath).toArray(String[]::new); - } - } - - @Override - public String[] getEngineConnTypeListFromDisk() { - return Arrays.stream(new File(getEngineConnsHome()).listFiles()) - .filter(file -> !file.isHidden() && file.isDirectory()) - .map(file -> file.getName()) - .toArray(String[]::new); - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java deleted file mode 100644 index c8ebc50633..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.localize; - -import org.apache.linkis.common.utils.ZipUtils; -import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException; -import org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary; - -import java.io.File; -import java.text.MessageFormat; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.NO_PERMISSION_FILE; - -public class DefaultEngineConnBmlResourceGenerator extends AbstractEngineConnBmlResourceGenerator { - - private static final Logger logger = - LoggerFactory.getLogger(DefaultEngineConnBmlResourceGenerator.class); - - public DefaultEngineConnBmlResourceGenerator() {} - - @Override - public Map generate(String engineConnType) { - String[] engineConnDistHomes = getEngineConnDistHomeList(engineConnType); - Map resultMap = new HashMap<>(); - for (String path : engineConnDistHomes) { - - File versionFile = new File(path); - logger.info("generate, versionFile:" + path); - if (!versionFile.isDirectory()) { - logger.warn("File is not dir {},skip to upload", path); - continue; - } - String key = versionFile.getName(); - - try { - EngineConnLocalizeResource[] engineConnLocalizeResources = - generateDir(versionFile.getPath()); - resultMap.put(key, engineConnLocalizeResources); - } catch (Throwable t) { - logger.error("Generate dir : " + path + " error, msg : " + t.getMessage(), t); - throw t; - } - } - - return resultMap; - } - - @Override - public EngineConnLocalizeResource[] generate(String engineConnType, String version) { - String path = getEngineConnDistHome(engineConnType, version); - return generateDir(path); - } - - private EngineConnLocalizeResource[] generateDir(String path) { - File distFile = new File(path); - if (!distFile.isDirectory()) { - logger.warn("File is not dir {},skip to upload", path); - throw new EngineConnPluginErrorException( - EngineconnCoreErrorCodeSummary.DIST_IRREGULAR_EXIST.getErrorCode(), - path + " is not dir, to delete this file then retry"); - } - logger.info("generateDir, distFile:" + path); - File[] validFiles = - distFile.listFiles( - f -> - !f.getName().endsWith(".zip") - || !new File(path, f.getName().replace(".zip", "")).exists()); - - return Arrays.stream(validFiles) - .map( - file -> { - if (file.isFile()) { - return new EngineConnLocalizeResourceImpl( - file.getPath(), file.getName(), file.lastModified(), file.length()); - } else { - File newFile = new File(path, file.getName() + ".zip"); - if (newFile.exists() && !newFile.delete()) { - throw new EngineConnPluginErrorException( - NO_PERMISSION_FILE.getErrorCode(), - MessageFormat.format(NO_PERMISSION_FILE.getErrorDesc(), newFile)); - } - - ZipUtils.fileToZip(file.getPath(), path, file.getName() + ".zip"); - // If it is a folder, the last update time here is the last update time of the - // folder, not the last update time of - // ZIP.(如果是文件夹，这里的最后更新时间，采用文件夹的最后更新时间，而不是ZIP的最后更新时间.) - return new EngineConnLocalizeResourceImpl( - newFile.getPath(), newFile.getName(), file.lastModified(), newFile.length()); - } - }) - .toArray(EngineConnLocalizeResource[]::new); - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java deleted file mode 100644 index 4ca366e8bf..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.localize; - -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EngineConnLocalizeResourceImpl implements EngineConnLocalizeResource { - private static final Logger logger = - LoggerFactory.getLogger(EngineConnLocalizeResourceImpl.class); - - private final String filePath; - private final String fileName; - private final long lastModified; - private final long fileSize; - - public EngineConnLocalizeResourceImpl( - String filePath, String fileName, long lastModified, long fileSize) { - this.filePath = filePath; - this.fileName = fileName; - this.lastModified = lastModified; - this.fileSize = fileSize; - } - - @Override - public InputStream getFileInputStream() { - try { - return new FileInputStream(filePath); - } catch (FileNotFoundException e) { - logger.warn("getFileInputStream failed filePath:[{}]", filePath, e); - } - return null; - } - - public String filePath() { - return filePath; - } - - public String fileName() { - return fileName; - } - - public long lastModified() { - return lastModified; - } - - public long fileSize() { - return fileSize; - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java deleted file mode 100644 index 807daa2c97..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -import org.apache.linkis.engineplugin.server.loader.EngineConnPluginsLoaderFactory; -import org.apache.linkis.manager.am.exception.AMErrorCode; -import org.apache.linkis.manager.am.util.LinkisUtils; -import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException; -import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder; -import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest; -import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; -import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResourceGenerator; -import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder; -import org.apache.linkis.manager.engineplugin.common.loader.entity.EngineConnPluginInstance; -import org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.rpc.message.annotation.Receiver; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Component -public class DefaultEngineConnLaunchService implements EngineConnLaunchService { - - private static final Logger logger = - LoggerFactory.getLogger(DefaultEngineConnLaunchService.class); - - @Autowired private EngineConnResourceGenerator engineConnResourceGenerator; - - private EngineConnLaunchBuilder getEngineLaunchBuilder( - EngineTypeLabel engineTypeLabel, EngineConnBuildRequest engineBuildRequest) { - final EngineConnPluginInstance engineConnPluginInstance; - try { - engineConnPluginInstance = - EngineConnPluginsLoaderFactory.getEngineConnPluginsLoader() - .getEngineConnPlugin(engineTypeLabel); - } catch (Exception e) { - throw new EngineConnPluginErrorException( - AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorCode(), - AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorDesc()); - } - final EngineConnLaunchBuilder builder = - engineConnPluginInstance.plugin().getEngineConnLaunchBuilder(); - if (builder instanceof JavaProcessEngineConnLaunchBuilder) { - ((JavaProcessEngineConnLaunchBuilder) builder) - .setEngineConnResourceGenerator(engineConnResourceGenerator); - } - builder.setBuildRequest(engineBuildRequest); - return builder; - } - - @Override - @Receiver - public EngineConnLaunchRequest createEngineConnLaunchRequest( - EngineConnBuildRequest engineBuildRequest) { - final Optional engineTypeOption = - engineBuildRequest.labels().stream() - .filter(label -> label instanceof EngineTypeLabel) - .map(label -> (EngineTypeLabel) label) - .findFirst(); - - if (!engineTypeOption.isPresent()) { - throw new EngineConnPluginErrorException( - EngineconnCoreErrorCodeSummary.ETL_REQUESTED.getErrorCode(), - EngineconnCoreErrorCodeSummary.ETL_REQUESTED.getErrorDesc()); - } - - final EngineTypeLabel engineTypeLabel = engineTypeOption.get(); - return LinkisUtils.tryCatch( - () -> getEngineLaunchBuilder(engineTypeLabel, engineBuildRequest).buildEngineConn(), - (Throwable t) -> { - logger.error( - String.format( - "Failed to createEngineConnLaunchRequest(%s)", engineBuildRequest.ticketId()), - t); - throw new EngineConnPluginErrorException( - EngineconnCoreErrorCodeSummary.FAILED_CREATE_ELR.getErrorCode(), - String.format( - "%s, %s", - EngineconnCoreErrorCodeSummary.FAILED_CREATE_ELR.getErrorDesc(), - ExceptionUtils.getRootCauseMessage(t))); - }); - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java deleted file mode 100644 index 2c0496c071..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -import org.apache.linkis.engineplugin.server.loader.EngineConnPluginsLoaderFactory; -import org.apache.linkis.manager.am.exception.AMErrorCode; -import org.apache.linkis.manager.am.exception.AMErrorException; -import org.apache.linkis.manager.common.entity.resource.NodeResource; -import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException; -import org.apache.linkis.manager.engineplugin.common.loader.entity.EngineConnPluginInstance; -import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory; -import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceRequest; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.rpc.message.annotation.Receiver; - -import org.springframework.stereotype.Component; - -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.ETL_REQUESTED; - -@Component -public class DefaultEngineConnResourceFactoryService implements EngineConnResourceFactoryService { - - private static final Logger logger = - LoggerFactory.getLogger(DefaultEngineConnResourceFactoryService.class); - - @Override - public EngineResourceFactory getResourceFactoryBy(EngineTypeLabel engineType) { - final EngineConnPluginInstance engineConnPluginInstance; - try { - engineConnPluginInstance = - EngineConnPluginsLoaderFactory.getEngineConnPluginsLoader() - .getEngineConnPlugin(engineType); - } catch (Exception e) { - logger.warn("getResourceFactory failed engineType:{}", engineType, e); - throw new AMErrorException( - AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorCode(), - AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorDesc()); - } - return engineConnPluginInstance.plugin().getEngineResourceFactory(); - } - - @Override - @Receiver - public NodeResource createEngineResource(final EngineResourceRequest engineResourceRequest) { - logger.info(String.format("To invoke createEngineResource %s", engineResourceRequest)); - final Optional engineTypeOption = - engineResourceRequest.labels().stream() - .filter(label -> label instanceof EngineTypeLabel) - .map(label -> (EngineTypeLabel) label) - .findFirst(); - - if (!engineTypeOption.isPresent()) { - throw new EngineConnPluginErrorException( - ETL_REQUESTED.getErrorCode(), ETL_REQUESTED.getErrorDesc()); - } - - final EngineTypeLabel engineTypeLabel = engineTypeOption.get(); - return getResourceFactoryBy(engineTypeLabel).createEngineResource(engineResourceRequest); - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java deleted file mode 100644 index a33b1afde8..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -import org.apache.linkis.bml.client.BmlClient; -import org.apache.linkis.bml.client.BmlClientFactory; -import org.apache.linkis.bml.protocol.BmlUpdateResponse; -import org.apache.linkis.bml.protocol.BmlUploadResponse; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration; -import org.apache.linkis.engineplugin.server.dao.EngineConnBmlResourceDao; -import org.apache.linkis.engineplugin.server.entity.EngineConnBmlResource; -import org.apache.linkis.engineplugin.server.localize.EngineConnBmlResourceGenerator; -import org.apache.linkis.engineplugin.server.localize.EngineConnLocalizeResource; -import org.apache.linkis.manager.common.protocol.bml.BmlResource; -import org.apache.linkis.manager.common.protocol.bml.BmlResource.BmlResourceVisibility; -import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException; -import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResource; -import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants; -import org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary; -import org.apache.linkis.rpc.message.annotation.Receiver; - -import org.apache.commons.lang3.StringUtils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; - -import java.text.MessageFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION; - -@Component -public class DefaultEngineConnResourceService extends EngineConnResourceService { - private static final Logger logger = - LoggerFactory.getLogger(DefaultEngineConnResourceService.class); - - @Autowired private EngineConnBmlResourceGenerator engineConnBmlResourceGenerator; - - @Autowired private EngineConnBmlResourceDao engineConnBmlResourceDao; - - private final BmlClient bmlClient = BmlClientFactory.createBmlClient(); - private boolean isRefreshing = false; - - @PostConstruct - @Override - public void init() { - if ((boolean) EngineConnPluginConfiguration.ENGINE_CONN_DIST_LOAD_ENABLE.getValue()) { - logger.info("Start to refresh all engineconn plugins when inited."); - refreshAll(false, false); - } - } - - private BmlResource uploadToBml(final EngineConnLocalizeResource localizeResource) { - final BmlUploadResponse response = - bmlClient.uploadResource( - Utils.getJvmUser(), localizeResource.fileName(), localizeResource.getFileInputStream()); - final BmlResource bmlResource = new BmlResource(); - bmlResource.setResourceId(response.resourceId()); - bmlResource.setVersion(response.version()); - return bmlResource; - } - - private BmlResource uploadToBml( - final EngineConnLocalizeResource localizeResource, final String resourceId) { - final BmlUpdateResponse response = - bmlClient.updateResource( - Utils.getJvmUser(), - resourceId, - localizeResource.fileName(), - localizeResource.getFileInputStream()); - final BmlResource bmlResource = new BmlResource(); - bmlResource.setResourceId(response.resourceId()); - bmlResource.setVersion(response.version()); - return bmlResource; - } - - @Override - public void refreshAll(boolean iswait, boolean force) { - if (!isRefreshing) { - synchronized (this) { - if (!isRefreshing) { - - final Runnable refreshTask = - new Runnable() { - @Override - public void run() { - isRefreshing = true; - logger.info("Try to initialize the dist resources of all EngineConns. "); - String[] engineConnTypeList = - engineConnBmlResourceGenerator.getEngineConnTypeListFromDisk(); - for (String engineConnType : engineConnTypeList) { - try { - logger.info( - "Try to initialize all versions of {}EngineConn.", engineConnType); - Map version2Localize = - engineConnBmlResourceGenerator.generate(engineConnType); - for (Map.Entry entry : - version2Localize.entrySet()) { - logger.info( - "Try to initialize {}EngineConn-{}.", engineConnType, entry.getKey()); - refresh(entry.getValue(), engineConnType, entry.getKey(), force); - } - - } catch (Exception t) { - if (!iswait - && EngineConnPluginConfiguration.ENABLED_BML_UPLOAD_FAILED_EXIT - .getValue()) { - logger.error("Failed to upload engine conn to bml, now exit!", t); - System.exit(1); - } - logger.error("Failed to upload engine conn to bml", t); - } - } - isRefreshing = false; - } - }; - Future future = Utils.defaultScheduler().submit(refreshTask); - - if (iswait) { - try { - future.get(); - } catch (InterruptedException | ExecutionException e) { - logger.info("DefaultEngineConnResourceService refreshTask execution failed", e); - } - } else { - logger.info("DefaultEngineConnResourceService IsRefreshing EngineConns..."); - } - } - } - } - } - - @Receiver - public boolean refreshAll(final RefreshAllEngineConnResourceRequest engineConnRefreshAllRequest) { - logger.info("Start to refresh all engineconn plugins."); - refreshAll(true, false); - return true; - } - - @Receiver - @Override - public boolean refresh( - final RefreshEngineConnResourceRequest engineConnRefreshRequest, final boolean force) { - final String engineConnType = engineConnRefreshRequest.getEngineConnType(); - final String version = engineConnRefreshRequest.getVersion(); - if ("*".equals(version) || StringUtils.isEmpty(version)) { - logger.info("Try to refresh all versions of {}EngineConn.", engineConnType); - Map version2Localize = - engineConnBmlResourceGenerator.generate(engineConnType); - for (Map.Entry entry : version2Localize.entrySet()) { - logger.info("Try to initialize {}EngineConn-{}.", engineConnType, entry.getKey()); - refresh(entry.getValue(), engineConnType, entry.getKey(), force); - } - - } else { - logger.info("Try to refresh {}EngineConn-{}.", engineConnType, version); - EngineConnLocalizeResource[] localize = - engineConnBmlResourceGenerator.generate(engineConnType, version); - refresh(localize, engineConnType, version, force); - } - return true; - } - - private void refresh( - final EngineConnLocalizeResource[] localize, - final String engineConnType, - final String version, - final boolean force) { - final List engineConnBmlResources = - engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version); - - if (Stream.of(localize) - .filter( - localizeResource -> - StringUtils.equals( - LaunchConstants.ENGINE_CONN_CONF_DIR_NAME() + ".zip", - localizeResource.fileName()) - || StringUtils.equals( - LaunchConstants.ENGINE_CONN_LIB_DIR_NAME() + ".zip", - localizeResource.fileName())) - .count() - < 2) { - - throw new EngineConnPluginErrorException( - EngineconnCoreErrorCodeSummary.LIB_CONF_DIR_NECESSARY.getErrorCode(), - MessageFormat.format( - EngineconnCoreErrorCodeSummary.LIB_CONF_DIR_NECESSARY.getErrorDesc(), - engineConnType)); - } - - for (EngineConnLocalizeResource localizeResource : localize) { - - Optional resource = - engineConnBmlResources.stream() - .filter(r -> r.getFileName().equals(localizeResource.fileName())) - .findFirst(); - if (!resource.isPresent()) { - logger.info( - "Ready to upload a new bmlResource for {}EngineConn-{}. path: {}", - engineConnType, - version, - localizeResource.fileName()); - final BmlResource bmlResource = uploadToBml(localizeResource); - final EngineConnBmlResource engineConnBmlResource = new EngineConnBmlResource(); - engineConnBmlResource.setBmlResourceId(bmlResource.getResourceId()); - engineConnBmlResource.setBmlResourceVersion(bmlResource.getVersion()); - engineConnBmlResource.setCreateTime(new Date()); - engineConnBmlResource.setLastUpdateTime(new Date()); - engineConnBmlResource.setEngineConnType(engineConnType); - engineConnBmlResource.setFileName(localizeResource.fileName()); - engineConnBmlResource.setFileSize(localizeResource.fileSize()); - engineConnBmlResource.setLastModified(localizeResource.lastModified()); - engineConnBmlResource.setVersion(version); - engineConnBmlResourceDao.save(engineConnBmlResource); - } else { - boolean isChanged = - resource.get().getFileSize() != localizeResource.fileSize() - || resource.get().getLastModified() != localizeResource.lastModified(); - - if (isChanged || (!isChanged && force)) { - if (!isChanged && force) { - logger.info( - "The file has no change in {}EngineConn-{}, path: {}, but force to refresh", - engineConnType, - version, - localizeResource.fileName()); - } - logger.info( - "Ready to upload a refreshed bmlResource for {}EngineConn-{}. path: {}", - engineConnType, - version, - localizeResource.fileName()); - final EngineConnBmlResource engineConnBmlResource = resource.get(); - final BmlResource bmlResource = - uploadToBml(localizeResource, engineConnBmlResource.getBmlResourceId()); - engineConnBmlResource.setBmlResourceVersion(bmlResource.getVersion()); - engineConnBmlResource.setLastUpdateTime(new Date()); - engineConnBmlResource.setFileSize(localizeResource.fileSize()); - engineConnBmlResource.setLastModified(localizeResource.lastModified()); - engineConnBmlResourceDao.update(engineConnBmlResource); - } else { - logger.info( - "The file has no change in {}EngineConn-{}, path: {}", - engineConnType, - version, - localizeResource.fileName()); - } - } - } - } - - @Receiver - @Override - public EngineConnResource getEngineConnBMLResources( - final GetEngineConnResourceRequest engineConnBMLResourceRequest) { - final String engineConnType = engineConnBMLResourceRequest.getEngineConnType(); - final String version = engineConnBMLResourceRequest.getVersion(); - - List engineConnBmlResources = - engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version); - if (engineConnBmlResources.size() == 0 - && (boolean) EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue()) { - logger.info("Try to get engine conn bml resource with prefex v"); - engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, "v" + version); - } - - Optional confBmlResourceMap = - engineConnBmlResources.stream() - .filter( - r -> r.getFileName().equals(LaunchConstants.ENGINE_CONN_CONF_DIR_NAME() + ".zip")) - .map(this::parseToBmlResource) - .findFirst(); - Optional libBmlResourceMap = - engineConnBmlResources.stream() - .filter( - r -> r.getFileName().equals(LaunchConstants.ENGINE_CONN_LIB_DIR_NAME() + ".zip")) - .map(this::parseToBmlResource) - .findFirst(); - - if (!confBmlResourceMap.isPresent() || !libBmlResourceMap.isPresent()) { - throw new EngineConnPluginErrorException( - EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION.getErrorCode(), - EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION.getErrorDesc()); - } - final BmlResource confBmlResource = confBmlResourceMap.get(); - final BmlResource libBmlResource = libBmlResourceMap.get(); - BmlResource[] otherBmlResources = - engineConnBmlResources.stream() - .filter( - r -> - !r.getFileName().equals(LaunchConstants.ENGINE_CONN_CONF_DIR_NAME() + ".zip") - || r.getFileName() - .equals(LaunchConstants.ENGINE_CONN_LIB_DIR_NAME() + ".zip")) - .map(this::parseToBmlResource) - .toArray(BmlResource[]::new); - - return new EngineConnResource() { - @Override - public BmlResource getConfBmlResource() { - return confBmlResource; - } - - @Override - public BmlResource getLibBmlResource() { - return libBmlResource; - } - - @Override - public BmlResource[] getOtherBmlResources() { - return otherBmlResources; - } - }; - } - - private BmlResource parseToBmlResource(final EngineConnBmlResource engineConnBmlResource) { - final BmlResource bmlResource = new BmlResource(); - bmlResource.setFileName(engineConnBmlResource.getFileName()); - bmlResource.setOwner(Utils.getJvmUser()); - bmlResource.setResourceId(engineConnBmlResource.getBmlResourceId()); - bmlResource.setVersion(engineConnBmlResource.getBmlResourceVersion()); - bmlResource.setVisibility(BmlResourceVisibility.Public); - return bmlResource; - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java deleted file mode 100644 index 62f7aea661..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -import org.apache.linkis.protocol.message.RequestMethod; -import org.apache.linkis.protocol.message.RequestProtocol; - -public abstract class EngineConnResourceRequest implements RequestProtocol, RequestMethod { - - private String engineConnType; - private String version; - - private boolean force; - - public String getEngineConnType() { - return engineConnType; - } - - public void setEngineConnType(String engineConnType) { - this.engineConnType = engineConnType; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public boolean getForce() { - return force; - } - - public void setForce(boolean force) { - this.force = force; - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java deleted file mode 100644 index 5f2fb6cf69..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResource; -import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResourceGenerator; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; - -public abstract class EngineConnResourceService implements EngineConnResourceGenerator { - - public abstract void init(); - - public abstract void refreshAll(boolean wait, boolean force); - - public abstract boolean refresh( - RefreshEngineConnResourceRequest engineConnRefreshRequest, boolean force); - - public abstract EngineConnResource getEngineConnBMLResources( - GetEngineConnResourceRequest engineConnBMLResourceRequest); - - @Override - public EngineConnResource getEngineConnBMLResources(EngineTypeLabel engineTypeLabel) { - GetEngineConnResourceRequest engineConnBMLResourceRequest = new GetEngineConnResourceRequest(); - engineConnBMLResourceRequest.setEngineConnType(engineTypeLabel.getEngineType()); - engineConnBMLResourceRequest.setVersion(engineTypeLabel.getVersion()); - return getEngineConnBMLResources(engineConnBMLResourceRequest); - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java deleted file mode 100644 index 046a0dd7ec..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -public class GetEngineConnResourceRequest extends EngineConnResourceRequest { - @Override - public String method() { - return "/enginePlugin/engineConn/getResource"; - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java deleted file mode 100644 index 6bd41d2599..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -import org.apache.linkis.protocol.message.RequestMethod; -import org.apache.linkis.protocol.message.RequestProtocol; - -public class RefreshAllEngineConnResourceRequest implements RequestProtocol, RequestMethod { - @Override - public String method() { - return "/enginePlugin/engineConn/refreshAll"; - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java deleted file mode 100644 index f518a72059..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.server.service; - -public class RefreshEngineConnResourceRequest extends EngineConnResourceRequest { - @Override - public String method() { - return "/enginePlugin/engineConn/refresh"; - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java index 603641fa78..803151d534 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java @@ -21,7 +21,6 @@ import org.apache.linkis.bml.client.BmlClientFactory; import org.apache.linkis.bml.protocol.BmlResourceVersionsResponse; import org.apache.linkis.bml.protocol.Version; -import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.common.utils.ZipUtils; import org.apache.linkis.engineplugin.server.dao.EngineConnBmlResourceDao; import org.apache.linkis.engineplugin.server.entity.EngineConnBmlResource; @@ -29,8 +28,6 @@ import org.apache.linkis.engineplugin.server.restful.EnginePluginRestful; import org.apache.linkis.engineplugin.server.service.EnginePluginAdminService; import org.apache.linkis.engineplugin.vo.EnginePluginBMLVo; -import org.apache.linkis.manager.am.exception.AMErrorCode; -import org.apache.linkis.manager.am.exception.AMErrorException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -40,7 +37,6 @@ import java.io.FileOutputStream; import java.io.InputStream; import java.io.OutputStream; -import java.text.MessageFormat; import java.util.List; import com.github.pagehelper.PageHelper; @@ -83,11 +79,6 @@ public List getTypeList() { @Override public void deleteEnginePluginBML(String ecType, String version, String username) { List allEngineConnBmlResource = null; - if (ecType != null && SecurityUtils.containsRelativePath(ecType)) { - throw new AMErrorException( - AMErrorCode.EC_PLUGIN_ERROR.getErrorCode(), - MessageFormat.format(AMErrorCode.EC_PLUGIN_ERROR.getErrorDesc(), ecType)); - } try { allEngineConnBmlResource = engineConnBmlResourceDao.getAllEngineConnBmlResource(ecType, version); @@ -97,20 +88,13 @@ public void deleteEnginePluginBML(String ecType, String version, String username engineConnBmlResourceDao.delete(engineConnBmlResource); }); String engineConnsHome = defaultEngineConnBmlResourceGenerator.getEngineConnsHome(); - File file = new File(engineConnsHome + "/" + ecType); - if (file.exists()) { deleteDir(file); log.info("file {} delete success", ecType); } } catch (Exception e) { - log.warn( - "deleteEnginePluginBML failed ecType:[{}] version:[{}] username:[{}]", - ecType, - version, - username, - e); + e.printStackTrace(); } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java index 9fc63ebcc8..064d61a6fb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java @@ -24,5 +24,6 @@ public class LinkisManagerApplication { public static void main(String[] args) throws ReflectiveOperationException { LinkisBaseServerApp.main(args); + // DataWorkCloudApplication.main(args); } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java index 80b433b3f2..5164542445 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -23,10 +23,54 @@ import org.apache.linkis.manager.common.entity.enumeration.MaintainType; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; public class AMConfiguration { + // The configuration key for the YARN queue name. + public static final String YARN_QUEUE_NAME_CONFIG_KEY = "wds.linkis.rm.yarnqueue"; + + // Identifier for cross-queue tasks. + public static final String CROSS_QUEUE = "crossQueue"; + + // Identifier for across-cluster tasks. + public static final String ACROSS_CLUSTER_TASK = "acrossClusterTask"; + + // Identifier for priority clusters. + public static final String PRIORITY_CLUSTER = "priorityCluster"; + + // Target identifier for distinguishing target clusters. + public static final String PRIORITY_CLUSTER_TARGET = "bdp"; + + // Origin identifier for distinguishing source clusters. + public static final String PRIORITY_CLUSTER_ORIGIN = "bdap"; + + // Configuration key for the target cluster CPU threshold. + public static final String TARGET_CPU_THRESHOLD = "targetCPUThreshold"; + + // Configuration key for the target cluster memory threshold. + public static final String TARGET_MEMORY_THRESHOLD = "targetMemoryThreshold"; + + // Configuration key for the target cluster CPU percentage threshold. + public static final String TARGET_CPU_PERCENTAGE_THRESHOLD = "targetCPUPercentageThreshold"; + + // Configuration key for the target cluster memory percentage threshold. + public static final String TARGET_MEMORY_PERCENTAGE_THRESHOLD = "targetMemoryPercentageThreshold"; + + // Configuration key for the origin cluster CPU percentage threshold. + public static final String ORIGIN_CPU_PERCENTAGE_THRESHOLD = "originCPUPercentageThreshold"; + + // Configuration key for the origin cluster memory percentage threshold. + public static final String ORIGIN_MEMORY_PERCENTAGE_THRESHOLD = "originMemoryPercentageThreshold"; + + public static final double ACROSS_CLUSTER_TOTAL_MEMORY_PERCENTAGE_THRESHOLD = + CommonVars.apply("linkis.yarn.across.cluster.memory.threshold", 0.8).getValue(); + + public static final double ACROSS_CLUSTER_TOTAL_CPU_PERCENTAGE_THRESHOLD = + CommonVars.apply("linkis.yarn.across.cluster.cpu.threshold", 0.8).getValue(); + public static final CommonVars ECM_ADMIN_OPERATIONS = CommonVars.apply("wds.linkis.governance.admin.operations", ""); @@ -39,50 +83,35 @@ public class AMConfiguration { public static final CommonVars ENGINE_REUSE_MAX_TIME = CommonVars.apply("wds.linkis.manager.am.engine.reuse.max.time", new TimeType("5m")); - public static final CommonVars ENGINE_REUSE_COUNT_LIMIT = - CommonVars.apply("wds.linkis.manager.am.engine.reuse.count.limit", 2); - - public static final CommonVars NODE_STATUS_HEARTBEAT_TIME = - CommonVars.apply("wds.linkis.manager.am.node.heartbeat", new TimeType("3m")); - - public static final CommonVars NODE_HEARTBEAT_MAX_UPDATE_TIME = - CommonVars.apply("wds.linkis.manager.am.node.heartbeat", new TimeType("5m")); + public static final Integer ENGINE_REUSE_COUNT_LIMIT = + CommonVars.apply("wds.linkis.manager.am.engine.reuse.count.limit", 2).getValue(); public static final CommonVars DEFAULT_NODE_OWNER = CommonVars.apply("wds.linkis.manager.am.default.node.owner", "hadoop"); - public static final CommonVars STOP_ENGINE_WAIT = - CommonVars.apply("wds.linkis.manager.am.stop.engine.wait", new TimeType("5m")); - - public static final CommonVars STOP_EM_WAIT = - CommonVars.apply("wds.linkis.manager.am.stop.em.wait", new TimeType("5m")); - - public static final CommonVars EM_LABEL_INIT_WAIT = - CommonVars.apply("wds.linkis.manager.am.em.label.init.wait", new TimeType("5m")); - public static final CommonVars EM_NEW_WAIT_MILLS = CommonVars.apply("wds.linkis.manager.am.em.new.wait.mills", 1000 * 60L); - public static final CommonVars ENGINECONN_DEBUG_ENABLED = - CommonVars.apply("wds.linkis.engineconn.debug.mode.enable", false); - public static final CommonVars MULTI_USER_ENGINE_TYPES = CommonVars.apply( "wds.linkis.multi.user.engine.types", - "jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase,doris"); + "jdbc,es,presto,io_file,appconn,openlookeng,trino,jobserver,nebula,hbase,doris"); public static final CommonVars ALLOW_BATCH_KILL_ENGINE_TYPES = CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); + public static final CommonVars UNALLOW_BATCH_KILL_ENGINE_TYPES = + CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "trino,appconn,io_file"); public static final CommonVars MULTI_USER_ENGINE_USER = CommonVars.apply("wds.linkis.multi.user.engine.user", getDefaultMultiEngineUser()); public static final CommonVars ENGINE_LOCKER_MAX_TIME = CommonVars.apply("wds.linkis.manager.am.engine.locker.max.time", 1000 * 60 * 5); - public static final CommonVars AM_CAN_RETRY_LOGS = + public static final String AM_CAN_RETRY_LOGS = CommonVars.apply( - "wds.linkis.manager.am.can.retry.logs", "already in use;Cannot allocate memory"); + "wds.linkis.manager.am.can.retry.logs", "already in use;Cannot allocate memory") + .getValue(); public static final int ASK_ENGINE_ASYNC_MAX_THREAD_SIZE = CommonVars.apply("wds.linkis.ecm.launch.max.thread.size", 200).getValue(); @@ -102,11 +131,56 @@ public class AMConfiguration { public static final Boolean NODE_SELECT_HOTSPOT_EXCLUSION_RULE = CommonVars.apply("linkis.node.select.hotspot.exclusion.rule.enable", true).getValue(); + public static final boolean EC_REUSE_WITH_RESOURCE_RULE_ENABLE = + CommonVars.apply("linkis.ec.reuse.with.resource.rule.enable", false).getValue(); + + public static final String EC_REUSE_WITH_RESOURCE_WITH_ECS = + CommonVars.apply("linkis.ec.reuse.with.resource.with.ecs", "spark,hive,shell,python") + .getValue(); + + public static final String SUPPORT_CLUSTER_RULE_EC_TYPES = + CommonVars.apply("linkis.support.cluster.rule.ec.types", "").getValue(); + + public static final boolean HIVE_CLUSTER_EC_EXECUTE_ONCE_RULE_ENABLE = + CommonVars.apply("linkis.hive.cluster.ec.execute.once.rule.enable", true).getValue(); + + public static final String LONG_LIVED_LABEL = + CommonVars.apply("linkis.label.node.long.lived.label.keys", "tenant|yarnCluster").getValue(); + + public static final String TMP_LIVED_LABEL = + CommonVars.apply("linkis.label.node.tmp.lived.label.keys", "taskId").getValue(); + + public static final boolean COMBINED_WITHOUT_YARN_DEFAULT = + CommonVars.apply("linkis.combined.without.yarn.default", true).getValue(); + + public static final Map AM_ENGINE_ASK_MAX_NUMBER = new HashMap<>(); + + static { + String keyValue = + CommonVars.apply("linkis.am.engine.ask.max.number", "appconn=5,trino=10").getValue(); + String[] keyValuePairs = keyValue.split(","); + for (String pair : keyValuePairs) { + String[] array = pair.split("="); + if (array.length != 2) { + throw new IllegalArgumentException( + "linkis.am.engine.ask.max.number value is illegal, value is " + pair); + } else { + AM_ENGINE_ASK_MAX_NUMBER.put(array[0], Integer.parseInt(array[1])); + } + } + } + + public static final boolean AM_ECM_RESET_RESOURCE = + CommonVars.apply("linkis.am.ecm.reset.resource.enable", true).getValue(); + + public static final boolean AM_USER_RESET_RESOURCE = + CommonVars.apply("linkis.am.user.reset.resource.enable", true).getValue(); + public static String getDefaultMultiEngineUser() { String jvmUser = Utils.getJvmUser(); return String.format( - "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\",doris:\"%s\", hbase:\"%s\",io_file:\"root\"}", - jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); + "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\",doris:\"%s\", hbase:\"%s\", jobserver:\"%s\",io_file:\"root\"}", + jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); } public static boolean isMultiUserEngine(String engineType) { @@ -123,4 +197,14 @@ public static boolean isAllowKilledEngineType(String engineType) { Arrays.stream(allowBatchKillEngine).filter(e -> e.equalsIgnoreCase(engineType)).findFirst(); return findResult.isPresent(); } + + public static boolean isUnAllowKilledEngineType(String engineType) { + String[] unAllowBatchKillEngine = + AMConfiguration.UNALLOW_BATCH_KILL_ENGINE_TYPES.getValue().split(","); + Optional findResult = + Arrays.stream(unAllowBatchKillEngine) + .filter(e -> e.equalsIgnoreCase(engineType)) + .findFirst(); + return findResult.isPresent(); + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java deleted file mode 100644 index 96d28a185b..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.conf; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class ApplicationManagerSpringConfiguration { - - @ConditionalOnMissingBean - @Bean - public EngineConnConfigurationService getDefaultEngineConnConfigurationService() { - return new DefaultEngineConnConfigurationService(); - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java index 039704351e..fa9843d955 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java @@ -18,7 +18,9 @@ package org.apache.linkis.manager.am.conf; import org.apache.linkis.common.conf.Configuration; -import org.apache.linkis.governance.common.protocol.conf.*; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfigWithGlobalConfig; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryGlobalConfig; +import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; import org.apache.linkis.protocol.CacheableProtocol; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java index 3575491087..1492c6569f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java @@ -17,11 +17,13 @@ package org.apache.linkis.manager.am.conf; -import org.apache.linkis.manager.am.util.LinkisUtils; +import org.apache.linkis.common.utils.LinkisUtils; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.springframework.stereotype.Component; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Component class DefaultEngineConnConfigurationService implements EngineConnConfigurationService { private static final Logger logger = diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java index c3a35f7921..bce581a2e9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java @@ -28,12 +28,6 @@ public class ManagerMonitorConf { public static final CommonVars NODE_HEARTBEAT_MAX_UPDATE_TIME = CommonVars.apply("wds.linkis.manager.am.node.heartbeat", new TimeType("12m")); - public static final CommonVars ENGINE_KILL_TIMEOUT = - CommonVars.apply("wds.linkis.manager.am.engine.kill.timeout", new TimeType("2m")); - - public static final CommonVars EM_KILL_TIMEOUT = - CommonVars.apply("wds.linkis.manager.am.em.kill.timeout", new TimeType("2m")); - public static final CommonVars MANAGER_MONITOR_ASYNC_POLL_SIZE = CommonVars.apply("wds.linkis.manager.monitor.async.poll.size", 5); @@ -42,4 +36,7 @@ public class ManagerMonitorConf { public static final CommonVars ECM_HEARTBEAT_MAX_UPDATE_TIME = CommonVars.apply("wds.linkis.manager.am.ecm.heartbeat", new TimeType("5m")); + + public static final CommonVars ACROSS_QUEUES_RESOURCE_SHOW_SWITCH_ON = + CommonVars.apply("wds.linkis.manager.across.resource.show.switch.on", false); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java index 4736433ada..c7620157a1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java @@ -23,7 +23,6 @@ import org.apache.linkis.manager.common.entity.metrics.NodeOverLoadInfo; import org.apache.linkis.manager.common.entity.metrics.NodeTaskInfo; import org.apache.linkis.manager.common.entity.node.AMNode; -import org.apache.linkis.manager.service.common.metrics.MetricsConverter; import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.commons.lang3.StringUtils; @@ -55,7 +54,7 @@ public NodeTaskInfo parseTaskInfo(NodeMetrics nodeMetrics) { return taskInfo; } } catch (IOException e) { - logger.error("parse task info failed", e); + logger.warn("parse task info failed", e); } } return null; @@ -68,7 +67,7 @@ public NodeHealthyInfo parseHealthyInfo(NodeMetrics nodeMetrics) { try { return BDPJettyServerHelper.jacksonJson().readValue(healthyInfo, NodeHealthyInfo.class); } catch (IOException e) { - logger.error("parse healthy info failed", e); + logger.warn("parse healthy info failed", e); } } return null; @@ -81,7 +80,7 @@ public NodeOverLoadInfo parseOverLoadInfo(NodeMetrics nodeMetrics) { try { return BDPJettyServerHelper.jacksonJson().readValue(overLoad, NodeOverLoadInfo.class); } catch (IOException e) { - logger.error("parse over load info failed", e); + logger.warn("parse over load info failed", e); } } return null; @@ -97,7 +96,7 @@ public String convertTaskInfo(NodeTaskInfo nodeTaskInfo) { try { return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeTaskInfo); } catch (JsonProcessingException e) { - logger.error("convert task info failed", e); + logger.warn("convert task info failed", e); } return null; } @@ -107,7 +106,7 @@ public String convertHealthyInfo(NodeHealthyInfo nodeHealthyInfo) { try { return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeHealthyInfo); } catch (JsonProcessingException e) { - logger.error("convert healthy info failed", e); + logger.warn("convert healthy info failed", e); } return null; } @@ -117,7 +116,7 @@ public String convertOverLoadInfo(NodeOverLoadInfo nodeOverLoadInfo) { try { return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeOverLoadInfo); } catch (JsonProcessingException e) { - logger.error("convert over load info failed", e); + logger.warn("convert over load info failed", e); } return null; } @@ -129,7 +128,9 @@ public int convertStatus(NodeStatus nodeStatus) { @Override public AMNode fillMetricsToNode(AMNode amNode, NodeMetrics metrics) { - if (metrics == null) return amNode; + if (metrics == null) { + return amNode; + } amNode.setNodeStatus(parseStatus(metrics)); amNode.setNodeTaskInfo(parseTaskInfo(metrics)); amNode.setNodeHealthyInfo(parseHealthyInfo(metrics)); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/metrics/MetricsConverter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/MetricsConverter.java similarity index 97% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/metrics/MetricsConverter.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/MetricsConverter.java index acc46510cd..e84b577f45 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/metrics/MetricsConverter.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/MetricsConverter.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.metrics; +package org.apache.linkis.manager.am.converter; import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.manager.common.entity.enumeration.NodeHealthy; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java index cc8997d857..c05768739c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorCode.java @@ -32,8 +32,7 @@ public enum AMErrorCode implements LinkisErrorCode { ASK_ENGINE_ERROR_RETRY(210005, "Ask engine error, retry(请求引擎失败，重试)"), - EC_OPERATE_ERROR(210006, "Failed to execute operation(引擎操作失败)"), - EC_PLUGIN_ERROR(210007, "ECType {0} contains RelativePath"); + EC_OPERATE_ERROR(210006, "Failed to execute operation(引擎操作失败)"); private final int errorCode; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorException.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorException.java index 4f3badbb9a..727fcc3133 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorException.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/exception/AMErrorException.java @@ -17,10 +17,9 @@ package org.apache.linkis.manager.am.exception; -import org.apache.linkis.common.exception.ExceptionLevel; -import org.apache.linkis.common.exception.LinkisRuntimeException; +import org.apache.linkis.common.exception.ErrorException; -public class AMErrorException extends LinkisRuntimeException { +public class AMErrorException extends ErrorException { public AMErrorException(int errCode, String desc) { super(errCode, desc); @@ -30,9 +29,4 @@ public AMErrorException(int errCode, String desc, Throwable t) { this(errCode, desc); this.initCause(t); } - - @Override - public ExceptionLevel getLevel() { - return ExceptionLevel.ERROR; - } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java index a0928db981..13e45832c8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java @@ -21,7 +21,6 @@ import org.apache.linkis.manager.label.entity.em.EMInstanceLabel; import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; -import org.apache.linkis.manager.service.common.label.LabelChecker; import org.springframework.stereotype.Component; @@ -45,7 +44,6 @@ public boolean checkEMLabel(List> labelList) { @Override public boolean checkCorrespondingLabel(List> labelList, Class... clazz) { - // TODO: 是否需要做子类的判断 List> classes = Arrays.asList(clazz); return labelList.stream() .filter(Objects::nonNull) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java index 5fa8e7db12..8820bc0be8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java @@ -18,7 +18,9 @@ package org.apache.linkis.manager.am.label; import org.apache.linkis.governance.common.conf.GovernanceCommonConf; -import org.apache.linkis.manager.label.entity.*; +import org.apache.linkis.manager.label.entity.EMNodeLabel; +import org.apache.linkis.manager.label.entity.EngineNodeLabel; +import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.node.AliasServiceInstanceLabel; import org.apache.linkis.manager.service.common.label.LabelFilter; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/label/LabelChecker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/LabelChecker.java similarity index 95% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/label/LabelChecker.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/LabelChecker.java index 87dab1f333..0c1e27d086 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/label/LabelChecker.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/LabelChecker.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.label; +package org.apache.linkis.manager.am.label; import org.apache.linkis.manager.label.entity.Label; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java index 4fbe6894f4..b8ed766072 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java @@ -38,8 +38,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static scala.collection.JavaConverters.*; - @Component public class MultiUserEngineReuseLabelChooser implements EngineReuseLabelChooser { private static final Logger logger = diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java index 00adf1492b..fe93a03411 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java @@ -17,13 +17,13 @@ package org.apache.linkis.manager.am.locker; +import org.apache.linkis.manager.am.pointer.NodePointerBuilder; import org.apache.linkis.manager.common.entity.node.AMEngineNode; import org.apache.linkis.manager.common.entity.node.EngineNode; import org.apache.linkis.manager.common.protocol.RequestEngineLock; import org.apache.linkis.manager.common.protocol.RequestEngineUnlock; import org.apache.linkis.manager.common.protocol.RequestManagerUnlock; import org.apache.linkis.manager.common.protocol.engine.EngineLockType; -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; import org.apache.linkis.rpc.message.annotation.Receiver; import org.springframework.beans.factory.annotation.Autowired; @@ -58,14 +58,14 @@ public void releaseLock(RequestManagerUnlock requestManagerUnlock) { logger.info( String.format( "client%s Start to unlock engine %s", - requestManagerUnlock.getClientInstance(), requestManagerUnlock.getEngineInstance())); + requestManagerUnlock.clientInstance(), requestManagerUnlock.engineInstance())); AMEngineNode engineNode = new AMEngineNode(); - engineNode.setServiceInstance(requestManagerUnlock.getEngineInstance()); - releaseLock(engineNode, requestManagerUnlock.getLock()); + engineNode.setServiceInstance(requestManagerUnlock.engineInstance()); + releaseLock(engineNode, requestManagerUnlock.lock()); logger.info( String.format( "client%s Finished to unlock engine %s", - requestManagerUnlock.getClientInstance(), requestManagerUnlock.getEngineInstance())); + requestManagerUnlock.clientInstance(), requestManagerUnlock.engineInstance())); } catch (Exception e) { logger.error("release lock failed", e); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java index cf66e88e5d..691aa635a4 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java @@ -18,6 +18,8 @@ package org.apache.linkis.manager.am.manager; import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.am.converter.MetricsConverter; +import org.apache.linkis.manager.am.pointer.NodePointerBuilder; import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; import org.apache.linkis.manager.common.entity.node.*; import org.apache.linkis.manager.common.entity.persistence.PersistenceNodeEntity; @@ -30,8 +32,6 @@ import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; import org.apache.linkis.manager.rm.ResourceInfo; import org.apache.linkis.manager.rm.service.ResourceManager; -import org.apache.linkis.manager.service.common.metrics.MetricsConverter; -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -150,7 +150,7 @@ public EMNode[] getEMNodes(ScoreServiceInstance[] scoreServiceInstances) { .filter(metrics -> metrics.getServiceInstance().equals(emNode.getServiceInstance())) .findFirst(); Optional optionRMNode = - resourceInfo.getResourceInfo().stream() + resourceInfo.resourceInfo().stream() .filter(rmNode -> rmNode.getServiceInstance().equals(emNode.getServiceInstance())) .findFirst(); @@ -171,7 +171,7 @@ public EMNode getEM(ServiceInstance serviceInstance) { emNode.setOwner(node.getOwner()); emNode.setServiceInstance(node.getServiceInstance()); if (node instanceof PersistenceNodeEntity) { - emNode.setStartTime(((PersistenceNodeEntity) node).getStartTime()); + emNode.setStartTime(node.getStartTime()); } emNode.setMark(emNode.getMark()); metricsConverter.fillMetricsToNode(emNode, nodeMetricManagerPersistence.getNodeMetrics(emNode)); @@ -197,7 +197,7 @@ public void pauseEM(ServiceInstance serviceInstance) {} /** * 1. request engineManager to launch engine * - * @param engineBuildRequest + * @param engineConnLaunchRequest * @param emNode * @return */ diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java index 14d548ef77..02b143d5cd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java @@ -20,11 +20,12 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.common.exception.LinkisRetryException; import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.converter.MetricsConverter; import org.apache.linkis.manager.am.exception.AMErrorCode; import org.apache.linkis.manager.am.exception.AMErrorException; import org.apache.linkis.manager.am.locker.EngineNodeLocker; -import org.apache.linkis.manager.am.utils.DefaultRetryHandler; -import org.apache.linkis.manager.am.utils.RetryHandler; +import org.apache.linkis.manager.am.pointer.EngineNodePointer; +import org.apache.linkis.manager.am.pointer.NodePointerBuilder; import org.apache.linkis.manager.common.constant.AMConstant; import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; @@ -41,11 +42,12 @@ import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; import org.apache.linkis.manager.rm.ResourceInfo; import org.apache.linkis.manager.rm.service.ResourceManager; -import org.apache.linkis.manager.service.common.metrics.MetricsConverter; -import org.apache.linkis.manager.service.common.pointer.EngineNodePointer; -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; + +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import java.lang.reflect.UndeclaredThrowableException; @@ -104,9 +106,12 @@ public List listEngines(String user) { return nodes; } + @Retryable( + value = {feign.RetryableException.class, UndeclaredThrowableException.class}, + maxAttempts = 5, + backoff = @Backoff(delay = 10000)) @Override public EngineNode getEngineNodeInfo(EngineNode engineNode) { - /** Change the EngineNode to correspond to real-time requests?（修改为实时请求对应的EngineNode) */ EngineNodePointer engine = nodePointerBuilder.buildEngineNodePointer(engineNode); NodeHeartbeatMsg heartMsg = engine.getNodeHeartbeatMsg(); engineNode.setNodeHealthyInfo(heartMsg.getHealthyInfo()); @@ -155,7 +160,7 @@ public EngineNode switchEngine(EngineNode engineNode) { @Override public EngineNode reuseEngine(EngineNode engineNode) { EngineNode node = getEngineNodeInfo(engineNode); - if (!NodeStatus.isAvailable(node.getNodeStatus())) { + if (node == null || !NodeStatus.isAvailable(node.getNodeStatus())) { return null; } if (!NodeStatus.isLocked(node.getNodeStatus())) { @@ -183,22 +188,8 @@ public EngineNode reuseEngine(EngineNode engineNode) { */ @Override public EngineNode useEngine(EngineNode engineNode, long timeout) { - RetryHandler retryHandler = new DefaultRetryHandler(); - retryHandler.addRetryException(feign.RetryableException.class); - retryHandler.addRetryException(UndeclaredThrowableException.class); // wait until engine to be available - EngineNode node = retryHandler.retry(() -> getEngineNodeInfo(engineNode), "getEngineNodeInfo"); - long retryEndTime = System.currentTimeMillis() + 60 * 1000; - while ((node == null || !NodeStatus.isAvailable(node.getNodeStatus())) - && System.currentTimeMillis() < retryEndTime) { - node = retryHandler.retry(() -> getEngineNodeInfo(engineNode), "getEngineNodeInfo"); - try { - Thread.sleep(5 * 1000); - } catch (InterruptedException e) { - // ignore - } - } - + EngineNode node = getEngineNodeInfo(engineNode); if (node == null || !NodeStatus.isAvailable(node.getNodeStatus())) { return null; } @@ -216,8 +207,13 @@ public EngineNode useEngine(EngineNode engineNode, long timeout) { } } + @Override + public EngineNode useEngine(EngineNode engineNode) { + return useEngine(engineNode, AMConfiguration.ENGINE_LOCKER_MAX_TIME.getValue()); + } + /** - * Get detailed engine information from the persistence //TODO 是否增加owner到node + * Get detailed engine information from the persistence * * @param scoreServiceInstances * @return @@ -227,8 +223,9 @@ public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) if (scoreServiceInstances == null || scoreServiceInstances.length == 0) { return null; } + List scoreServiceInstancesList = Arrays.asList(scoreServiceInstances); EngineNode[] engineNodes = - Arrays.stream(scoreServiceInstances) + scoreServiceInstancesList.stream() .map( scoreServiceInstance -> { AMEngineNode engineNode = new AMEngineNode(); @@ -237,42 +234,48 @@ public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) return engineNode; }) .toArray(EngineNode[]::new); - // 1. add nodeMetrics 2 add RM info - ServiceInstance[] serviceInstances = - Arrays.stream(scoreServiceInstances) + + List serviceInstancesList = + scoreServiceInstancesList.stream() .map(ScoreServiceInstance::getServiceInstance) - .toArray(ServiceInstance[]::new); - ResourceInfo resourceInfo = resourceManager.getResourceInfo(serviceInstances); - - List nodeMetrics = - nodeMetricManagerPersistence.getNodeMetrics( - Arrays.stream(engineNodes).collect(Collectors.toList())); - Arrays.stream(engineNodes) - .forEach( - engineNode -> { - Optional optionMetrics = - nodeMetrics.stream() - .filter( - nodeMetric -> - nodeMetric - .getServiceInstance() - .equals(engineNode.getServiceInstance())) - .findFirst(); - - Optional optionRMNode = - resourceInfo.getResourceInfo().stream() - .filter( - resourceNode -> - resourceNode - .getServiceInstance() - .equals(engineNode.getServiceInstance())) - .findFirst(); - - optionMetrics.ifPresent( - metrics -> metricsConverter.fillMetricsToNode(engineNode, metrics)); - optionRMNode.ifPresent( - rmNode -> engineNode.setNodeResource(rmNode.getNodeResource())); - }); + .collect(Collectors.toList()); + + try { + ResourceInfo resourceInfo = + resourceManager.getResourceInfo(serviceInstancesList.toArray(new ServiceInstance[0])); + + if (serviceInstancesList.isEmpty()) { + throw new LinkisRetryException( + AMConstant.ENGINE_ERROR_CODE, "Service instances cannot be empty."); + } + + List nodeMetrics = + nodeMetricManagerPersistence.getNodeMetrics(Arrays.asList(engineNodes)); + + for (EngineNode engineNode : engineNodes) { + Optional optionMetrics = + nodeMetrics.stream() + .filter( + nodeMetric -> + nodeMetric.getServiceInstance().equals(engineNode.getServiceInstance())) + .findFirst(); + + Optional optionRMNode = + resourceInfo.resourceInfo().stream() + .filter( + resourceNode -> + resourceNode.getServiceInstance().equals(engineNode.getServiceInstance())) + .findFirst(); + + optionMetrics.ifPresent(metrics -> metricsConverter.fillMetricsToNode(engineNode, metrics)); + optionRMNode.ifPresent(rmNode -> engineNode.setNodeResource(rmNode.getNodeResource())); + } + } catch (Exception e) { + LinkisRetryException linkisRetryException = + new LinkisRetryException(AMConstant.ENGINE_ERROR_CODE, "Failed to process data."); + linkisRetryException.initCause(e); + throw linkisRetryException; + } return engineNodes; } @@ -349,14 +352,16 @@ public EngineNode getEngineNodeInfo(ServiceInstance serviceInstance) { AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorCode(), AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorDesc()); } - + NodeMetrics nodeMetric = nodeMetricManagerPersistence.getNodeMetrics(engineNode); if (engineNode.getNodeStatus() == null) { - NodeMetrics nodeMetric = nodeMetricManagerPersistence.getNodeMetrics(engineNode); - if (Objects.nonNull(nodeMetric) && Objects.nonNull(nodeMetric.getStatus())) { + if (null != nodeMetric && null != nodeMetric.getStatus()) { engineNode.setNodeStatus(NodeStatus.values()[nodeMetric.getStatus()]); } else { engineNode.setNodeStatus(NodeStatus.Starting); } + if (null != nodeMetric && StringUtils.isNotBlank(nodeMetric.getHeartBeatMsg())) { + engineNode.setEcMetrics(nodeMetric.getHeartBeatMsg()); + } } return engineNode; } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java index 8cb8362ecc..cf540036a3 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java @@ -20,6 +20,7 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.node.Node; import org.apache.linkis.manager.common.entity.node.ScoreServiceInstance; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; @@ -32,11 +33,11 @@ public interface EMNodeManager { void emRegister(EMNode emNode); - List listEngines(EMNode emNode); + List listEngines(EMNode emNode); - List listUserEngines(EMNode emNode, String user); + List listUserEngines(EMNode emNode, String user); - List listUserNodes(String user); + List listUserNodes(String user); /** * Get detailed em information from the persistence diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java index ce79d79c7e..7c3f64efee 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java @@ -65,5 +65,7 @@ void updateEngineStatus( EngineNode useEngine(EngineNode engineNode, long timeout); + EngineNode useEngine(EngineNode engineNode); + EngineOperateResponse executeOperation(EngineNode engineNode, EngineOperateRequest request); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java index 80390d0883..fc0a1fca24 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java @@ -17,12 +17,10 @@ package org.apache.linkis.manager.am.pointer; -import org.apache.linkis.common.exception.WarnException; import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; import org.apache.linkis.manager.common.entity.node.Node; import org.apache.linkis.manager.common.protocol.node.*; import org.apache.linkis.manager.label.entity.Label; -import org.apache.linkis.manager.service.common.pointer.NodePointer; import org.apache.linkis.rpc.Sender; public abstract class AbstractNodePointer implements NodePointer { @@ -39,12 +37,8 @@ protected Sender getSender() { @Override public NodeStatus getNodeStatus() { Sender sender = getSender(); - try { - ResponseNodeStatus responseStatus = (ResponseNodeStatus) sender.ask(new RequestNodeStatus()); - return responseStatus.getNodeStatus(); - } catch (WarnException e) { - throw e; - } + ResponseNodeStatus responseStatus = (ResponseNodeStatus) sender.ask(new RequestNodeStatus()); + return responseStatus.getNodeStatus(); } /** @@ -55,12 +49,8 @@ public NodeStatus getNodeStatus() { @Override public NodeHeartbeatMsg getNodeHeartbeatMsg() { Sender sender = getSender(); - try { - NodeHeartbeatMsg heartbeatMsg = (NodeHeartbeatMsg) sender.ask(new NodeHeartbeatRequest()); - return heartbeatMsg; - } catch (WarnException e) { - throw e; - } + NodeHeartbeatMsg heartbeatMsg = (NodeHeartbeatMsg) sender.ask(new NodeHeartbeatRequest()); + return heartbeatMsg; } /** diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java index 1458680c98..07097fcb0b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java @@ -27,7 +27,6 @@ import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; -import org.apache.linkis.manager.service.common.pointer.EMNodPointer; import org.apache.linkis.server.BDPJettyServerHelper; import org.slf4j.Logger; @@ -73,28 +72,39 @@ public EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest) @Override public void stopEngine(EngineStopRequest engineStopRequest) { - Object result = getSender().ask(engineStopRequest); - if (result instanceof EngineStopResponse) { - EngineStopResponse engineStopResponse = (EngineStopResponse) result; - if (!engineStopResponse.getStopStatus()) { - logger.info( - "Kill engine : " - + engineStopRequest.getServiceInstance().toString() - + " failed, because " - + engineStopResponse.getMsg() - + " . Will ask engine to suicide."); + try { + Object result = getSender().ask(engineStopRequest); + if (result instanceof EngineStopResponse) { + EngineStopResponse engineStopResponse = (EngineStopResponse) result; + if (!engineStopResponse.getStopStatus()) { + logger.info( + "Kill engine : " + + engineStopRequest.getServiceInstance().toString() + + " failed, because " + + engineStopResponse.getMsg() + + " . Will ask engine to suicide."); + } else { + logger.info( + "Succeed to kill engine " + engineStopRequest.getServiceInstance().toString() + "."); + } } else { - logger.info( - "Succeed to kill engine " + engineStopRequest.getServiceInstance().toString() + "."); + logger.warn( + "Ask em : " + + getNode().getServiceInstance().toString() + + " to kill engine : " + + engineStopRequest.getServiceInstance().toString() + + " failed, response is : " + + BDPJettyServerHelper.gson().toJson(result) + + "."); } - } else { + } catch (Exception e) { logger.warn( "Ask em : " + getNode().getServiceInstance().toString() + " to kill engine : " + engineStopRequest.getServiceInstance().toString() - + " failed, response is : " - + BDPJettyServerHelper.gson().toJson(result) + + " failed, exception is : " + + e.getMessage() + "."); } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java index 21f86f83d1..cdbbcbbf09 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java @@ -24,7 +24,6 @@ import org.apache.linkis.manager.common.protocol.ResponseEngineLock; import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; -import org.apache.linkis.manager.service.common.pointer.EngineNodePointer; import java.util.Optional; @@ -50,14 +49,11 @@ public Optional lockEngine(RequestEngineLock requestEngineLock) { Object result = getSender().ask(requestEngineLock); if (result instanceof ResponseEngineLock) { ResponseEngineLock responseEngineLock = (ResponseEngineLock) result; - if (responseEngineLock.getLockStatus()) { - return Optional.of(responseEngineLock.getLock()); + if (responseEngineLock.lockStatus()) { + return Optional.of(responseEngineLock.lock()); } else { logger.info( - "Failed to get locker," - + node.getServiceInstance() - + ": " - + responseEngineLock.getMsg()); + "Failed to get locker," + node.getServiceInstance() + ": " + responseEngineLock.msg()); return Optional.empty(); } } else { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java index 03e0bc8087..58ec1ae940 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java @@ -19,9 +19,6 @@ import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; -import org.apache.linkis.manager.service.common.pointer.EMNodPointer; -import org.apache.linkis.manager.service.common.pointer.EngineNodePointer; -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; import org.springframework.stereotype.Component; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/EMNodPointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EMNodPointer.java similarity index 96% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/EMNodPointer.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EMNodPointer.java index 6574563003..a85bf6eaca 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/EMNodPointer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EMNodPointer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.pointer; +package org.apache.linkis.manager.am.pointer; import org.apache.linkis.manager.common.entity.node.EngineNode; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EngineNodePointer.java similarity index 96% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EngineNodePointer.java index cc24b746af..8be00a09c2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EngineNodePointer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.pointer; +package org.apache.linkis.manager.am.pointer; import org.apache.linkis.manager.common.protocol.RequestEngineLock; import org.apache.linkis.manager.common.protocol.RequestEngineUnlock; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/NodePointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointer.java similarity index 96% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/NodePointer.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointer.java index ca27bf8194..e5d519873e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/NodePointer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.pointer; +package org.apache.linkis.manager.am.pointer; import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; import org.apache.linkis.manager.common.entity.node.Node; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointerBuilder.java similarity index 94% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointerBuilder.java index e993a724b1..649e142399 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointerBuilder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.pointer; +package org.apache.linkis.manager.am.pointer; import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index c3c14a8094..70caae1a8d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -71,6 +71,7 @@ public class ECResourceInfoRestfulApi { public Message getECInfo( HttpServletRequest req, @RequestParam(value = "ticketid") String ticketid) throws AMErrorException { + logger.info("ticked: {} get ec info", ticketid); ECResourceInfoRecord ecResourceInfoRecord = ecResourceInfoService.getECResourceInfoRecord(ticketid); String userName = ModuleUserUtils.getOperationUser(req, "getECInfo ticketid:" + ticketid); @@ -87,7 +88,7 @@ public Message getECInfo( @ApiImplicitParams({ @ApiImplicitParam(name = "ticketid", required = true, dataType = "String", value = "ticket id") }) - @RequestMapping(path = "/delete/{ticketid}", method = RequestMethod.DELETE) + @RequestMapping(path = "/delete/{ticketid}}", method = RequestMethod.DELETE) public Message deleteECInfo(HttpServletRequest req, @PathVariable("ticketid") String ticketid) throws AMErrorException { ECResourceInfoRecord ecResourceInfoRecord = @@ -113,6 +114,7 @@ public Message deleteECInfo(HttpServletRequest req, @PathVariable("ticketid") St @ApiImplicitParam(name = "startDate", dataType = "String", value = "start date"), @ApiImplicitParam(name = "endDate", dataType = "String", value = "end date"), @ApiImplicitParam(name = "engineType", dataType = "String", value = "engine type"), + @ApiImplicitParam(name = "status", dataType = "String", value = "engine status"), @ApiImplicitParam(name = "pageNow", dataType = "String", value = "page now"), @ApiImplicitParam(name = "pageSize", dataType = "String", value = "page size") }) @@ -131,6 +133,7 @@ public Message queryEcrHistory( defaultValue = "#{new java.util.Date()}") Date endDate, @RequestParam(value = "engineType", required = false) String engineType, + @RequestParam(value = "status", required = false) String status, @RequestParam(value = "pageNow", required = false, defaultValue = "1") Integer pageNow, @RequestParam(value = "pageSize", required = false, defaultValue = "20") Integer pageSize) { String username = SecurityFilter.getLoginUsername(req); @@ -138,6 +141,7 @@ public Message queryEcrHistory( instance = ECResourceInfoUtils.strCheckAndDef(instance, null); String creatorUser = ECResourceInfoUtils.strCheckAndDef(creator, null); engineType = ECResourceInfoUtils.strCheckAndDef(engineType, null); + status = ECResourceInfoUtils.strCheckAndDef(status, null); if (null != creatorUser && !ECResourceInfoUtils.checkNameValid(creatorUser)) { return Message.error("Invalid creator : " + creatorUser); } @@ -148,7 +152,7 @@ public Message queryEcrHistory( calendar.set(Calendar.SECOND, 0); startDate = calendar.getTime(); } - if (Configuration.isAdmin(username)) { + if (Configuration.isJobHistoryAdmin(username)) { username = null; if (StringUtils.isNotBlank(creatorUser)) { username = creatorUser; @@ -161,7 +165,7 @@ public Message queryEcrHistory( try { queryTasks = ecResourceInfoService.getECResourceInfoRecordList( - instance, endDate, startDate, username, engineType); + instance, endDate, startDate, username, engineType, status); queryTasks.forEach( info -> { ECResourceInfoRecordVo ecrHistroryListVo = new ECResourceInfoRecordVo(); @@ -188,25 +192,51 @@ public Message queryEcrHistory( @ApiImplicitParam(name = "creators", dataType = "Array", required = true, value = "creators"), @ApiImplicitParam(name = "engineTypes", dataType = "Array", value = "engine type"), @ApiImplicitParam(name = "statuss", dataType = "Array", value = "statuss"), + @ApiImplicitParam(name = "queueName", dataType = "String", value = "queueName"), + @ApiImplicitParam(name = "ecInstances", dataType = "Array", value = "ecInstances"), + @ApiImplicitParam(name = "crossCluster", dataType = "String", value = "crossCluster"), }) @RequestMapping(path = "/ecList", method = RequestMethod.POST) public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNode) { + String username = ModuleUserUtils.getOperationUser(req, "ecList"); + String token = ModuleUserUtils.getToken(req); + // check special admin token + if (StringUtils.isNotBlank(token)) { + if (!Configuration.isAdminToken(token)) { + logger.warn("Token:{} has no permission to query ecList.", token); + return Message.error("Token:" + token + " has no permission to query ecList."); + } + } else if (!Configuration.isAdmin(username)) { + logger.warn("User:{} has no permission to query ecList.", username); + return Message.error("User:" + username + " has no permission to query ecList."); + } JsonNode creatorsParam = jsonNode.get("creators"); JsonNode engineTypesParam = jsonNode.get("engineTypes"); JsonNode statussParam = jsonNode.get("statuss"); + JsonNode queueNameParam = jsonNode.get("queueName"); + JsonNode ecInstancesParam = jsonNode.get("ecInstances"); + JsonNode crossClusterParam = jsonNode.get("crossCluster"); - if (creatorsParam == null || creatorsParam.isNull() || creatorsParam.size() == 0) { - return Message.error("creators is null in the parameters of the request(请求参数中【creators】为空)"); - } + // if (creatorsParam == null || creatorsParam.isNull() || creatorsParam.size() == 0) { + // return Message.error("creators is null in the parameters of the + // request(请求参数中【creators】为空)"); + // } List creatorUserList = new ArrayList<>(); - try { - creatorUserList = - JsonUtils.jackson() - .readValue(creatorsParam.toString(), new TypeReference>() {}); - } catch (JsonProcessingException e) { - return Message.error("parameters:creators parsing failed(请求参数【creators】解析失败)"); + if (creatorsParam != null && !creatorsParam.isNull()) { + try { + creatorUserList = + JsonUtils.jackson() + .readValue(creatorsParam.toString(), new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:creators parsing failed(请求参数【creators】解析失败)"); + } + for (String creatorUser : creatorUserList) { + if (null != creatorUser && !ECResourceInfoUtils.checkNameValid(creatorUser)) { + return Message.error("Invalid creator: " + creatorUser); + } + } } List engineTypeList = new ArrayList<>(); @@ -230,34 +260,52 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod return Message.error("parameters:statuss parsing failed(请求参数【statuss】解析失败)"); } } - - String username = ModuleUserUtils.getOperationUser(req, "ecList"); - - String token = ModuleUserUtils.getToken(req); - // check special admin token - if (StringUtils.isNotBlank(token)) { - if (!Configuration.isAdminToken(token)) { - return Message.error("Token has no permission to query ecList."); + String queueName = ""; + if (queueNameParam != null && !queueNameParam.isNull()) { + try { + queueName = + JsonUtils.jackson() + .readValue(queueNameParam.toString(), new TypeReference() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:queueName parsing failed(请求参数【queueName】解析失败)"); } - } else if (!Configuration.isAdmin(username)) { - logger.warn("User:{} has no permission to query ecList.", username); - return Message.error("User:" + username + " has no permission to query ecList."); } - - for (String creatorUser : creatorUserList) { - if (null != creatorUser && !ECResourceInfoUtils.checkNameValid(creatorUser)) { - return Message.error("Invalid creator: " + creatorUser); + List ecInstancesList = new ArrayList<>(); + if (ecInstancesParam != null && !ecInstancesParam.isNull()) { + try { + ecInstancesList = + JsonUtils.jackson() + .readValue(ecInstancesParam.toString(), new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:instanceName parsing failed(请求参数【ecInstances】解析失败)"); + } + } + Boolean isCrossCluster = null; + if (crossClusterParam != null && !crossClusterParam.isNull()) { + try { + isCrossCluster = + JsonUtils.jackson() + .readValue(crossClusterParam.toString(), new TypeReference() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:crossCluster parsing failed(请求参数【crossCluster】解析失败)"); } } - logger.info( - "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}]", + "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}], queueName:{}, instanceNameList:{}", String.join(",", creatorUserList), String.join(",", engineTypeList), - String.join(",", statusStrList)); + String.join(",", statusStrList), + String.join(",", ecInstancesList), + queueNameParam); List> list = - ecResourceInfoService.getECResourceInfoList(creatorUserList, engineTypeList, statusStrList); + ecResourceInfoService.getECResourceInfoList( + creatorUserList, + engineTypeList, + statusStrList, + queueName, + ecInstancesList, + isCrossCluster); return Message.ok().data("ecList", list); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index 4d5cb480d3..fbebd6813b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -28,22 +28,35 @@ import org.apache.linkis.manager.am.service.ECResourceInfoService; import org.apache.linkis.manager.am.service.em.ECMOperateService; import org.apache.linkis.manager.am.service.em.EMInfoService; +import org.apache.linkis.manager.am.service.engine.DefaultEngineCreateService; import org.apache.linkis.manager.am.utils.AMUtils; +import org.apache.linkis.manager.am.vo.CanCreateECRes; import org.apache.linkis.manager.am.vo.EMNodeVo; import org.apache.linkis.manager.common.entity.enumeration.NodeHealthy; import org.apache.linkis.manager.common.entity.metrics.NodeHealthyInfo; import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; import org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord; -import org.apache.linkis.manager.common.protocol.OperateRequest; +import org.apache.linkis.manager.common.exception.RMErrorException; +import org.apache.linkis.manager.common.protocol.OperateRequest$; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest$; import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineCreateRequest; +import org.apache.linkis.manager.exception.PersistenceErrorException; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.UserModifiable; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; import org.apache.linkis.manager.label.exception.LabelErrorException; import org.apache.linkis.manager.label.service.NodeLabelService; +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator; +import org.apache.linkis.manager.persistence.LabelManagerPersistence; +import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.apache.linkis.manager.persistence.ResourceManagerPersistence; +import org.apache.linkis.manager.rm.external.service.ExternalResourceService; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -58,6 +71,7 @@ import javax.servlet.http.HttpServletRequest; +import java.text.MessageFormat; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -93,6 +107,17 @@ public class EMRestfulApi { @Autowired private ECMOperateService ecmOperateService; @Autowired private ECResourceInfoService ecResourceInfoService; + + @Autowired private ResourceManagerPersistence resourceManagerPersistence; + + @Autowired private LabelManagerPersistence labelManagerPersistence; + + @Autowired private ExternalResourceService externalResourceService; + + @Autowired private DefaultEngineCreateService defaultEngineCreateService; + + @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; + private LabelBuilderFactory stdLabelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); @@ -148,7 +173,7 @@ public Message listAllEMs( stream = stream.filter( em -> { - List> labels = em.getLabels(); + List