Merge pull request #824 from Kaali09/2.5.0_fix

DO-1771 updated the yarn role
Sunbird-Knowlg · Dec 18, 2019 · 67e15a1 · 67e15a1
2 parents e6939fc + fd4dc8e
commit 67e15a1
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 100 deletions.
diff --git a/ansible/roles/yarn/tasks/common.yml b/ansible/roles/yarn/tasks/common.yml
@@ -1,73 +1,76 @@
-- name: Download and extract hadoop tarball
-  unarchive:
-    src: "{{hadoop_download_url}}"
-    dest: "/usr/local/"
-    owner: hduser 
-    group: hadoop 
-    creates: "{{hadoop_yarn_home}}"
-    remote_src: yes
+- name: Common tasks for yarn master and slave
+  block:
+  - name: Download and extract hadoop tarball
+    unarchive:
+      src: "{{hadoop_download_url}}"
+      dest: "/usr/local/"
+      owner: hduser
+      group: hadoop
+      creates: "{{hadoop_yarn_home}}"
+      remote_src: yes
 
-- name: Creates symlink
-  file: 
-    src: /usr/local/hadoop-{{hadoop_version}}
-    dest: /usr/local/hadoop
-    owner: hduser 
-    group: hadoop 
-    state: link
+  - name: Creates symlink
+    file:
+      src: /usr/local/hadoop-{{hadoop_version}}
+      dest: /usr/local/hadoop
+      owner: hduser
+      group: hadoop
+      state: link
 
-- name: creating conf dir
-  file: 
-    path: "{{hadoop_yarn_home}}/conf"
-    owner: hduser 
-    group: hadoop 
-    recurse: yes 
-    state: directory
+  - name: creating conf dir
+    file:
+      path: "{{hadoop_yarn_home}}/conf"
+      owner: hduser
+      group: hadoop
+      recurse: yes
+      state: directory
 
-- name: Templating configs
-  template: 
-    src: "{{item}}" 
-    dest: "{{hadoop_yarn_home}}/conf/{{item}}"
-    owner: hduser 
-    group: hadoop
-  with_items:
-    - yarn-site.xml
-    - capacity-scheduler.xml
-    - core-site.xml
-    - log4j.properties
-    - hadoop-env.sh
+  - name: Templating configs
+    template:
+      src: "{{item}}"
+      dest: "{{hadoop_yarn_home}}/conf/{{item}}"
+      owner: hduser
+      group: hadoop
+    with_items:
+      - yarn-site.xml
+      - capacity-scheduler.xml
+      - core-site.xml
+      - log4j.properties
+      - hadoop-env.sh
 
-- name: Downloading artifacts
-  get_url: 
-    url: "http://search.maven.org/remotecontent?filepath=org/{{item}}"
-    dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/"
-    owner: hduser
-    group: hadoop
-  with_items:
-    - clapper/grizzled-slf4j_2.10/1.0.1/grizzled-slf4j_2.10-1.0.1.jar
-    - apache/samza/samza-yarn_2.10/0.8.0/samza-yarn_2.10-0.8.0.jar
-    - apache/samza/samza-core_2.10/0.8.0/samza-core_2.10-0.8.0.jar
+  - name: Downloading artifacts
+    get_url:
+      url: "http://search.maven.org/remotecontent?filepath=org/{{item}}"
+      dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/"
+      owner: hduser
+      group: hadoop
+    with_items:
+      - clapper/grizzled-slf4j_2.10/1.0.1/grizzled-slf4j_2.10-1.0.1.jar
+      - apache/samza/samza-yarn_2.10/0.8.0/samza-yarn_2.10-0.8.0.jar
+      - apache/samza/samza-core_2.10/0.8.0/samza-core_2.10-0.8.0.jar
 
-- name: Download and extract scala
-  unarchive:
-    src: "{{scala_download_url}}"
-    dest: "/usr/local/"
-    owner: hduser 
-    group: hadoop 
-    remote_src: yes
+  - name: Download and extract scala
+    unarchive:
+      src: "{{scala_download_url}}"
+      dest: "/usr/local/"
+      owner: hduser
+      group: hadoop
+      remote_src: yes
 
-- name: Creates symlink
-  file: 
-    src: "/usr/local/scala-{{scala_version}}" 
-    dest: /usr/local/scala 
-    owner: hduser 
-    group: hadoop 
-    state: link
+  - name: Creates symlink
+    file:
+      src: "/usr/local/scala-{{scala_version}}"
+      dest: /usr/local/scala
+      owner: hduser
+      group: hadoop
+      state: link
 
-- name: copying scala files
-  copy:
-    src: "/usr/local/scala-{{scala_version}}/lib/{{item}}" 
-    dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/"
-    remote_src: true
-  with_items:
-    - scala-compiler.jar
-    - scala-library.jar
+  - name: copying scala files
+    copy:
+      src: "/usr/local/scala-{{scala_version}}/lib/{{item}}"
+      dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/"
+      remote_src: true
+    with_items:
+      - scala-compiler.jar
+      - scala-library.jar
+  delegate_to: "{{slave|default(inventory_hostname)}}"
diff --git a/ansible/roles/yarn/tasks/main.yml b/ansible/roles/yarn/tasks/main.yml
@@ -12,53 +12,44 @@
 - include: common.yml
 
 - lineinfile:
-    dest: /home/hduser/.bashrc 
-    state: present 
-    regexp: '^HADOOP_YARN_HOME' 
+    dest: /home/hduser/.bashrc
+    state: present
+    regexp: '^HADOOP_YARN_HOME'
     line: 'HADOOP_YARN_HOME={{hadoop_yarn_home}}'
 
 - lineinfile:
-    dest: /home/hduser/.bashrc 
-    state: present 
-    regexp: '^HADOOP_CONF_DIR' 
+    dest: /home/hduser/.bashrc
+    state: present
+    regexp: '^HADOOP_CONF_DIR'
     line: 'HADOOP_CONF_DIR=$HADOOP_YARN_HOME/conf'
 
-- file: 
-    path: "/{{hadoop_yarn_home}}/conf/slaves" 
+- file:
+    path: "/{{hadoop_yarn_home}}/conf/slaves"
     state: touch
 
-- lineinfile: 
-    dest: "/{{hadoop_yarn_home}}/conf/slaves" 
-    state: present 
-    regexp: "^{{item}}" 
+- lineinfile:
+    dest: "/{{hadoop_yarn_home}}/conf/slaves"
+    state: present
+    regexp: "^{{item}}"
     line: "{{item}}"
   with_items: "{{yarn_slaves}}"
 
-# # Running common tasks in slaves
-# - name: Running common tasks in slaves
-#   include: common.yml
-#   delegate_to: "{{slave}}"
-#   with_items: "{{yarn_slaves}}"
-#   loop_control:
-#     loop_var: slave
-
-- name: Synchronizing "{{hadoop_yarn_home}}" with yarn slaves
-  synchronize:
-    src: "{{hadoop_yarn_home}}"
-    dest: "{{hadoop_yarn_home}}"
-    recursive: yes
-  delegate_to: "{{ item }}"
-  loop: "{{yarn_slaves}}"
+# Running common tasks in slaves
+- name: Running common tasks in slaves
+  include: common.yml
+  with_items: "{{yarn_slaves}}"
+  loop_control:
+    loop_var: slave
 
 - name: Copy truncate_files.sh
-  copy: 
-    src: truncate_logs.sh 
+  copy:
+    src: truncate_logs.sh
     dest: /usr/local/bin
     mode: 755
 
 - name: Add truncate logs to cron
-  cron: 
-    name: "Truncate yarn logs" 
-    minute: "0" 
-    job: "/usr/local/bin/truncate_logs.sh" 
+  cron:
+    name: "Truncate yarn logs"
+    minute: "0"
+    job: "/usr/local/bin/truncate_logs.sh"
     backup: yes
diff --git a/ansible/roles/yarn/templates/config.j2 b/ansible/roles/yarn/templates/config.j2
@@ -0,0 +1,3 @@
+Host {{item}}
+  IdentityFile ~/.ssh/hadoop_rsa
+  StrictHostKeyChecking no
diff --git a/ansible/roles/yarn/templates/hadoop-env.sh b/ansible/roles/yarn/templates/hadoop-env.sh
@@ -1 +1,2 @@
-export JAVA_HOME=/opt/jdk1.8.0_121/
+#export JAVA_HOME=/usr/lib/jvm/java-8-oracle
+export JAVA_HOME=/opt/jdk1.8.0_121
diff --git a/ansible/roles/yarn/templates/yarn-site.xml b/ansible/roles/yarn/templates/yarn-site.xml
@@ -37,6 +37,14 @@
   <property>
     <name>yarn.nodemanager.log.retain-seconds</name>
     <value>3600</value>
+  </property>
+    <property>
+    <name>yarn.nodemanager.recovery.enabled</name>
+    <value>true</value>
+  </property>
+  <property>
+      <name>yarn.nodemanager.address</name>
+    <value>0.0.0.0:45454</value>
   </property>
   <property>
     <name>yarn.nodemanager.resource.cpu-vcores</name>