BIGTOP-4305: Add MySQL to infra stack (apache#128)

lhpqaq · Dec 19, 2024 · afc7f35 · afc7f35
1 parent 25b4366
commit afc7f35
Show file tree

Hide file tree

Showing 18 changed files with 581 additions and 18 deletions.
diff --git a/bigtop-manager-bom/pom.xml b/bigtop-manager-bom/pom.xml
@@ -40,6 +40,7 @@
         <guava.version>32.1.1-jre</guava.version>
         <commons-collections4.version>4.4</commons-collections4.version>
         <commons-compress.version>1.26.2</commons-compress.version>
+        <xz.version>1.9</xz.version>
         <jwt.version>4.4.0</jwt.version>
         <janino.version>3.0.16</janino.version>
         <jaxb-runtime.version>4.0.4</jaxb-runtime.version>
@@ -124,6 +125,12 @@
                 <version>${commons-compress.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.tukaani</groupId>
+                <artifactId>xz</artifactId>
+                <version>${xz.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>com.google.guava</groupId>
                 <artifactId>guava</artifactId>

diff --git a/...ager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceAddJob.java b/...ager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ServiceAddJob.java
@@ -55,12 +55,12 @@ protected void injectBeans() {
 
     @Override
     protected void createStages() {
+        // Update cache files
+        super.createCacheStage();
+
         // Install components
         super.createAddStages();
 
-        // Update cache files after installed
-        super.createCacheStage();
-
         // Configure services
         super.createConfigureStages();
 

diff --git a/...ager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/configuration/common.xml b/...ager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/configuration/common.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+<configuration>
+    <property>
+        <name>root_password</name>
+        <value>root</value>
+        <display-name>Root Password</display-name>
+        <description>Password for MySQL root user.</description>
+    </property>
+</configuration>
diff --git a/...ager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/configuration/my.cnf.xml b/...ager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/configuration/my.cnf.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+<configuration>
+    <property>
+        <name>mysql_log_dir</name>
+        <value>/var/log/mysql</value>
+        <display-name>MySQL Log Dir</display-name>
+        <description>MySQL Log Dir</description>
+    </property>
+    <property>
+        <name>mysql_pid_dir</name>
+        <value>/var/run/mysql</value>
+        <display-name>MySQL PID Dir</display-name>
+        <description>MySQL Pid Dir</description>
+    </property>
+    <!-- my.cnf -->
+    <property>
+        <name>content</name>
+        <display-name>my.cnf template</display-name>
+        <description>This is the freemarker template for my.cnf file</description>
+        <value><![CDATA[
+[mysqld]
+port=3306
+basedir=${mysql_home}
+datadir=${mysql_home}/data
+pid-file = ${mysql_pid_dir}/mysqld.pid
+log-error = ${mysql_log_dir}/error.log
+general-log-file = ${mysql_log_dir}/general.log
+slow-query-log-file = ${mysql_log_dir}/slow.log
+log-bin = ${mysql_log_dir}/mysql-bin.log
+default-storage-engine=INNODB
+]]>
+        </value>
+        <attrs>
+            <type>longtext</type>
+        </attrs>
+    </property>
+</configuration>
diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/metainfo.xml b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/metainfo.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~    https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+-->
+
+<metainfo>
+    <service>
+        <name>mysql</name>
+        <display-name>MySQL</display-name>
+        <desc>
+            MySQL is an open source relational database management system (RDBMS) that’s used to store and manage data.
+            Its reliability, performance, scalability, and ease of use make MySQL a popular choice for developers.
+        </desc>
+        <version>8.0.40</version>
+        <user>mysql</user>
+
+        <components>
+            <component>
+                <name>mysql_server</name>
+                <display-name>MySQL Server</display-name>
+                <category>server</category>
+                <cardinality>1</cardinality>
+            </component>
+
+            <component>
+                <name>mysql_client</name>
+                <display-name>MySQL Client</display-name>
+                <category>client</category>
+                <cardinality>1+</cardinality>
+            </component>
+        </components>
+
+        <package-specifics>
+            <package-specific>
+                <architectures>
+                    <arch>x86_64</arch>
+                </architectures>
+                <packages>
+                    <package>
+                        <name>mysql-8.0.40-linux-glibc2.28-x86_64.tar.xz</name>
+                        <checksum>MD5:dcf2702f953d1969be44083f4f063f18</checksum>
+                    </package>
+                </packages>
+            </package-specific>
+            <package-specific>
+                <architectures>
+                    <arch>aarch64</arch>
+                </architectures>
+                <packages>
+                    <package>
+                        <name>mysql-8.0.40-linux-glibc2.28-aarch64.tar.xz</name>
+                        <checksum>MD5:a79f41ce62784a1a0e081c76116008de</checksum>
+                    </package>
+                </packages>
+            </package-specific>
+        </package-specifics>
+    </service>
+</metainfo>
diff --git a/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/order.json b/bigtop-manager-server/src/main/resources/stacks/infra/1.0.0/services/mysql/order.json
@@ -0,0 +1 @@
+{}
diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/pom.xml b/bigtop-manager-stack/bigtop-manager-stack-core/pom.xml
@@ -54,5 +54,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-compress</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>org.tukaani</groupId>
+            <artifactId>xz</artifactId>
+        </dependency>
     </dependencies>
 </project>
diff --git a/...r-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java b/...r-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/spi/param/BaseParams.java
@@ -18,7 +18,6 @@
  */
 package org.apache.bigtop.manager.stack.core.spi.param;
 
-import org.apache.bigtop.manager.common.enums.Command;
 import org.apache.bigtop.manager.common.message.entity.payload.CommandPayload;
 import org.apache.bigtop.manager.common.message.entity.pojo.PackageInfo;
 import org.apache.bigtop.manager.common.message.entity.pojo.PackageSpecificInfo;
@@ -55,10 +54,6 @@ public abstract class BaseParams implements Params {
     protected BaseParams(CommandPayload commandPayload) {
         this.commandPayload = commandPayload;
 
-        Command command = commandPayload.getCommand();
-        if (command == Command.ADD) {
-            return;
-        }
         // Global Parameters Injection
         Method[] declaredMethods = this.getClass().getDeclaredMethods();
         for (Method declaredMethod : declaredMethods) {

diff --git a/...ck-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java b/...ck-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballDownloader.java
@@ -108,10 +108,10 @@ private static Boolean downloadFile(String fileUrl, String saveDir) {
                 outputStream.close();
                 inputStream.close();
 
-                log.info("File downloaded: {}", saveFilePath);
+                log.info("File downloaded: [{}]", saveFilePath);
                 return true;
             } else {
-                log.info("No file to download. Server replied HTTP code: {}", responseCode);
+                log.info("No file to download. Server replied HTTP code: [{}]", responseCode);
                 return false;
             }
         } catch (Exception e) {

diff --git a/...ack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java b/...ack-core/src/main/java/org/apache/bigtop/manager/stack/core/tarball/TarballExtractor.java
@@ -23,6 +23,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -45,10 +46,12 @@ public static void extractTarball(String source, String dest, Integer skipLevels
         File tarball = new File(source);
         File destDir = new File(dest);
 
-        if (isTarGz(source)) {
-            extractTarGz(tarball, tis -> extract(tis, destDir, skipLevels));
-        } else if (isTar(source)) {
+        if (isTar(source)) {
             extractTar(tarball, tis -> extract(tis, destDir, skipLevels));
+        } else if (isTarGz(source)) {
+            extractTarGz(tarball, tis -> extract(tis, destDir, skipLevels));
+        } else if (isTarXz(source)) {
+            extractTarXz(tarball, tis -> extract(tis, destDir, skipLevels));
         } else {
             log.info("Unsupported file type: {}", source);
         }
@@ -125,11 +128,26 @@ private static void extractTarGz(File tarball, Function<TarArchiveInputStream, B
         }
     }
 
-    private static boolean isTarGz(String filePath) {
-        return filePath.endsWith(".tar.gz") || filePath.endsWith(".tgz");
+    private static void extractTarXz(File tarball, Function<TarArchiveInputStream, Boolean> func) {
+        try (InputStream fis = Files.newInputStream(tarball.toPath());
+                XZCompressorInputStream xzis = new XZCompressorInputStream(fis);
+                TarArchiveInputStream tis = new TarArchiveInputStream(xzis)) {
+            func.apply(tis);
+        } catch (Exception e) {
+            log.error("Error extracting tarball", e);
+            throw new StackException(e);
+        }
     }
 
     private static boolean isTar(String filePath) {
         return filePath.endsWith(".tar");
     }
+
+    private static boolean isTarGz(String filePath) {
+        return filePath.endsWith(".tar.gz") || filePath.endsWith(".tgz");
+    }
+
+    private static boolean isTarXz(String filePath) {
+        return filePath.endsWith(".tar.xz");
+    }
 }
diff --git a/...k-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java b/...k-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxFileUtils.java
@@ -254,6 +254,27 @@ public static void moveFile(String source, String dest) {
         }
     }
 
+    public static String readFile(String source) {
+        if (StringUtils.isBlank(source)) {
+            throw new StackException("source must not be empty");
+        }
+
+        List<String> builderParameters = new ArrayList<>();
+        builderParameters.add("cat");
+        builderParameters.add(source);
+
+        try {
+            ShellResult shellResult = sudoExecCmd(builderParameters);
+            if (shellResult.getExitCode() != MessageConstants.SUCCESS_CODE) {
+                throw new StackException(shellResult.getErrMsg());
+            }
+
+            return shellResult.getOutput();
+        } catch (IOException e) {
+            throw new StackException(e);
+        }
+    }
+
     /**
      * create symbolic link
      *

diff --git a/...ack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxOSUtils.java b/...ack-core/src/main/java/org/apache/bigtop/manager/stack/core/utils/linux/LinuxOSUtils.java
@@ -20,7 +20,6 @@
 
 import org.apache.bigtop.manager.common.shell.ShellExecutor;
 import org.apache.bigtop.manager.common.shell.ShellResult;
-import org.apache.bigtop.manager.common.utils.FileUtils;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -104,7 +103,7 @@ public static ShellResult checkProcess(String filepath) {
         }
         int pid;
         try {
-            pid = Integer.parseInt(FileUtils.readFile2Str(file).replaceAll("\r|\n", ""));
+            pid = Integer.parseInt(LinuxFileUtils.readFile(filepath).replaceAll("\r|\n", ""));
         } catch (Exception e) {
             log.warn("Pid file {} does not exist or does not contain a process id number, error", filepath, e);
             return new ShellResult(-1, "", "Component is not running");

diff --git a/...er-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java b/...er-stack-infra/src/main/java/org/apache/bigtop/manager/stack/infra/param/InfraParams.java
@@ -24,11 +24,44 @@
 import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import java.io.File;
+
 @Slf4j
 @NoArgsConstructor
 public abstract class InfraParams extends BaseParams {
 
     protected InfraParams(CommandPayload commandPayload) {
         super(commandPayload);
     }
+
+    /**
+     * Infra stack do not belong to any cluster, so we need to override this and provide a group name
+     *
+     * @return group name
+     */
+    @Override
+    public String group() {
+        return "infra";
+    }
+
+    /**
+     * Infra stack do not belong to any cluster, we cannot use stack home of cluster
+     *
+     * @return group name
+     */
+    @Override
+    public String stackHome() {
+        // Parent path of agent dir, which is bigtop-manager-agent/../
+        String parentPath = new File(InfraParams.class
+                        .getProtectionDomain()
+                        .getCodeSource()
+                        .getLocation()
+                        .getPath())
+                .getParentFile()
+                .getParentFile()
+                .getParentFile()
+                .getPath();
+
+        return parentPath + "/infras";
+    }
 }
diff --git a/...c/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaServerScript.java b/...c/main/java/org/apache/bigtop/manager/stack/infra/v1_0_0/grafana/GrafanaServerScript.java
@@ -54,7 +54,6 @@ public ShellResult start(Params params) {
         GrafanaParams grafanaParams = (GrafanaParams) params;
         String cmd = MessageFormat.format(
                 "nohup {0}/bin/grafana server --homepath {0} > {0}/nohup.out 2>&1 &", grafanaParams.serviceHome());
-        log.info(cmd);
         try {
             ShellResult shellResult = LinuxOSUtils.sudoExecCmd(cmd, grafanaParams.user());
             if (shellResult.getExitCode() != 0) {