CDH Installation using Cloudera Manager Command History

CDH Installation using Cloudera Manager Command History

1. Disable Transparent Huge Page (Add the Following Lines to /etc/rc.local file)
--In RHEL 6
[root@cdh-master ~]# echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled
[root@cdh-master ~]# echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag

--In RHEL 7
vi /etc/default/grub
GRUB_CMDLINE_LINUX="transparent_hugepage=never"
grub2-mkconfig -o /boot/grub2/grub.cfg
shutdown -r now

2. Change Virtual Machine Swappinnes
Open a file /etc/sysctl.conf and add the line vm.swappiness=10
Verify the same by listing contents of file /proc/sys/vm/swappiness

3. Disable Firewall
systemctl stop firewalld/iptables/ip6tables
systemctl disable firewalld/iptables/ip6tables

service stop iptables
service stop ip6tables

chkconfig iptables off
chkconfig ip6tables off

4. Disable SELinux
Edit file /etc/selinux/config

SELINUX=disabled

Note: Reboot is required to take effect

5. Assign Host Name
edit config file /etc/sysconfig/network
HOSTNAME=cdh-master.local

Update in /etc/hosts as well

192.168.1.5 cdh-master.local master
192.168.1.6 cdh-node1.local node1
192.168.1.7 cdh-node2.local node2
192.168.1.8 cdh-node3.local node3


6. Install and Start NTP Service
yum -y install ntp
service ntpd start
chkconfig ntpd on

in RHEL 7
systemctl start ntpd
systemctl enable ntpd

Edit /etc/ntp.conf file and add NTP server address
ntpdate -u <NTP Server IP/Name>
ntpdate -q 0.rhel.pool.ntp.org

7. Create Users and Groups
groupadd hadoop
useradd -g hadoop hduser

8. Create SSH Key file and create passwordless connection between master and clients

# ssh-keygen -t rsa

# ssh-copy-id 192.168.1.6
root@192.168.1.6's password:
Now try logging into the machine, with "ssh '192.168.1.6'", and check in:

  .ssh/authorized_keys

to make sure we haven't added extra keys that you weren't expecting.

Download Repo Tarball package
mkdir -p /var/www/html/cm/5.14.0/
cd /var/www/html/cm/5.14.0/
wget http://archive.cloudera.com/cm5/repo-as-tarball/5.14.0/cm5.14.0-centos7.tar.gz
tar -xvf cm5.14.0-centos7.tar.gz
yum install cloudera-manager-agent.x86_64 cloudera-manager-server cloudera-manager-daemons.x86_64 oracle-j2sdk1.7.x86_64 enterprise-debuginfo.x86_64

9. Setting JAVA_HOME Path

# cat /etc/default/cloudera-scm-server

export JAVA_HOME=/usr/java/jdk1.7.0_67-cloudera/

10. Install Custom Database MySQL

# yum install -y mysql-server mysql-connector-java
# chkconfig mysqld on
# service mysqld start

-- run the Script to secure MySQL Access
/usr/bin/mysql_secure_installation

# mysql -u root -pmysql
mysql> create user 'temp'@'%' identified by 'temp' ;
Query OK, 0 rows affected (0.00 sec)

mysql> grant all privileges on *.* to 'temp'@'%' with grant option ;
Query OK, 0 rows affected (0.00 sec)

--Initialize MySQL Database
[root@cdh-master ~]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -h localhost -u temp -ptemp --scm-host cdh-master.local scm temp temp

--After initialization verify database and its tables
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| scm                |
+--------------------+
3 rows in set (0.00 sec)

mysql> use scm;
Database changed
mysql> show tables;
Empty set (0.00 sec)

mysql> show tables;
+--------------------------------+
| Tables_in_scm                  |
+--------------------------------+
| AUDITS                         |
| CLIENT_CONFIGS                 |
| CLIENT_CONFIGS_TO_HOSTS        |
| CLUSTERS                       |
| CLUSTERS_AUD                   |
| CLUSTER_ACTIVATED_RELEASES     |
| CLUSTER_ACTIVATED_RELEASES_AUD |
| CLUSTER_MANAGED_RELEASES       |
| CLUSTER_UNDISTRIBUTED_RELEASES |
| CM_PEERS                       |
| CM_VERSION                     |
| COMMANDS                       |
| COMMANDS_DETAIL                |
| COMMAND_SCHEDULES              |
| CONFIGS                        |
| CONFIGS_AUD                    |
| CONFIG_CONTAINERS              |
| CREDENTIALS                    |
| DIAGNOSTICS_EVENTS             |
| EXTERNAL_ACCOUNTS              |
| EXTERNAL_ACCOUNTS_AUD          |
| GLOBAL_SETTINGS                |
| HOSTS                          |
| HOSTS_AUD                      |
| HOST_TEMPLATES                 |
| HOST_TEMPLATE_TO_ROLE_CONF_GRP |
| METRICS                        |
| PARCELS                        |
| PARCEL_COMPONENTS              |
| PROCESSES                      |
| PROCESSES_DETAIL               |
| PROCESS_ACTIVE_RELEASES        |
| RELEASES                       |
| RELEASES_AUD                   |
| REVISIONS                      |
| ROLES                          |
| ROLES_AUD                      |
| ROLE_CONFIG_GROUPS             |
| ROLE_CONFIG_GROUPS_AUD         |
| ROLE_STALENESS_STATUS          |
| SCHEMA_VERSION                 |
| SERVICES                       |
| SERVICES_AUD                   |
| SNAPSHOT_POLICIES              |
| USERS                          |
| USER_ROLES                     |
| USER_SETTINGS                  |
+--------------------------------+
47 rows in set (0.00 sec)


vi /etc/cloudera-scm-agent/config.ini
server_host=cdh-master.local

11. Install Required Packages on Agents

# yum install cloudera-manager-agent.x86_64 cloudera-manager-daemons.x86_64 oracle-j2sdk1.7.x86_64 enterprise-debuginfo.x86_64

-- Add CM Server Address in Agent configuration File
# cat /etc/cloudera-scm-agent/config.ini
# Hostname of the CM server.
server_host=cdh-master.local


-- Start Agent Service on Node1
# service cloudera-scm-agent start
# chkconfig cloudera-scm-agent on
12. Switch on CM Server Services and access Cloudera Manager

# service cloudera-scm-server start
# chkconfig cloudera-scm-server on

http://192.168.1.5:7180/cmf/login
admin
admin

Accept License Agreement
Click Continue
Click coniture
Select Hosts and Click Continue
Cluster Installation Page launches

[root@cdh-master html]# mkdir -p /var/www/html/parcels/cm/cdh5.14.0
[root@cdh-master html]# ls
cm  parcels
# cd /var/www/html/parcels/cm/cdh5.14.0/
# wget https://archive.cloudera.com/cdh5/parcels/5.14.0/CDH-5.14.0-1.cdh5.14.0.p0.24-el7.parcel
# wget https://archive.cloudera.com/cdh5/parcels/5.14.0/manifest.json

# mkdir -p /var/www/html/parcels/impala
# cd /var/www/html/parcels/impala
# wget https://archive.cloudera.com/impala/parcels/latest/IMPALA-2.1.0-1.impala2.0.0.p0.1995-el7.parcel
# wget https://archive.cloudera.com/impala/parcels/latest/manifest.json

# mkdir -p /var/www/html/parcels/kudu
# mkdir -p /var/www/html/parcels/kudu
# wget https://archive.cloudera.com/kudu/parcels/latest/KUDU-1.4.0-1.cdh5.12.2.p0.8-el7.parcel
# wget https://archive.cloudera.com/kudu/parcels/latest/manifest.json

# cd /var/www/html/parcels/accumulo/
# wget https://archive.cloudera.com/accumulo/parcels/1.4/ACCUMULO-1.4.4-1.cdh4.5.0.p0.65-el7.parcel
# wget https://archive.cloudera.com/accumulo/parcels/1.4/manifest.json

# cd /var/www/html/parcels/kafka/
# wget https://archive.cloudera.com/kafka/parcels/latest/KAFKA-3.1.0-1.3.1.0.p0.35-el7.parcel
# wget https://archive.cloudera.com/kafka/parcels/latest/manifest.json

# cd /var/www/html/parcels/spark/
# wget https://archive.cloudera.com/spark/parcels/latest/SPARK-0.9.0-1.cdh4.6.0.p0.98-el7.parcel
# wget https://archive.cloudera.com/spark/parcels/latest/manifest.json

# cd /var/www/html/parcels/search/
# wget https://archive.cloudera.com/search/parcels/latest/SOLR-1.3.0-1.cdh4.5.0.p0.9-el7.parcel
# wget https://archive.cloudera.com/search/parcels/latest/manifest.json

# cd /var/www/html/parcels/sqoop/
# wget https://archive.cloudera.com/sqoop-connectors/parcels/latest/SQOOP_NETEZZA_CONNECTOR-1.5.1c5-el7.parcel
# wget https://archive.cloudera.com/sqoop-connectors/parcels/latest/manifest.json

/var/www/html/
├── cm
│   ├── 5 -> 5.14.0
│   ├── 5.14 -> 5.14.0
│   ├── 5.14.0
│   │   ├── generated_index.html
│   │   ├── mirrors
│   │   ├── repodata
│   │   │   ├── filelists.xml.gz
│   │   │   ├── filelists.xml.gz.asc
│   │   │   ├── generated_index.html
│   │   │   ├── other.xml.gz
│   │   │   ├── other.xml.gz.asc
│   │   │   ├── primary.xml.gz
│   │   │   ├── primary.xml.gz.asc
│   │   │   ├── repomd.xml
│   │   │   └── repomd.xml.asc
│   │   └── RPMS
│   │       ├── generated_index.html
│   │       ├── noarch
│   │       │   └── generated_index.html
│   │       └── x86_64
│   │           ├── cloudera-manager-agent-5.14.0-1.cm5140.p0.25.el7.x86_64.rpm
│   │           ├── cloudera-manager-daemons-5.14.0-1.cm5140.p0.25.el7.x86_64.rpm
│   │           ├── cloudera-manager-server-5.14.0-1.cm5140.p0.25.el7.x86_64.rpm
│   │           ├── cloudera-manager-server-db-2-5.14.0-1.cm5140.p0.25.el7.x86_64.rpm
│   │           ├── enterprise-debuginfo-5.14.0-1.cm5140.p0.25.el7.x86_64.rpm
│   │           ├── generated_index.html
│   │           ├── jdk-6u31-linux-amd64.rpm
│   │           └── oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm
│   ├── cloudera-cm.repo
│   ├── generated_index.html
│   └── RPM-GPG-KEY-cloudera
└── parcels
    ├── accumulo
    │   ├── ACCUMULO-1.4.4-1.cdh4.5.0.p0.65-el7.parcel
    │   └── manifest.json
    ├── cm
    │   └── cdh5.14.0
    │       ├── CDH-5.14.0-1.cdh5.14.0.p0.24-el7.parcel
    │       └── manifest.json
    ├── impala
    │   ├── IMPALA-2.1.0-1.impala2.0.0.p0.1995-el7.parcel
    │   └── manifest.json
    ├── kafka
    │   ├── KAFKA-3.1.0-1.3.1.0.p0.35-el7.parcel
    │   └── manifest.json
    ├── kudu
    │   ├── KUDU-1.4.0-1.cdh5.12.2.p0.8-el7.parcel
    │   └── manifest.json
    ├── search
    │   ├── manifest.json
    │   ├── manifest.json.1
    │   └── SOLR-1.3.0-1.cdh4.5.0.p0.9-el7.parcel
    ├── spark
    │   ├── manifest.json
    │   └── SPARK-0.9.0-1.cdh4.6.0.p0.98-el7.parcel
    └── sqoop
        ├── manifest.json
        ├── SQOOP_NETEZZA_CONNECTOR-1.5.1c5-el7.parcel
        └── SQOOP_TERADATA_CONNECTOR-1.7c5-el7.parcel

18 directories, 42 files

Click continue


in Node1
# yum install mysql-server mysql-connector-java -y
# service mysqld start

# mysql -u root -pmysql
mysql> create user 'temp'@'%' identified by 'temp' ;
Query OK, 0 rows affected (0.00 sec)

[root@cdh-master parcel]# mysql -u aravi -p
Enter password:
Welcome to the MariaDB monitor.  Commands end with ; or \g.
Your MariaDB connection id is 383
Server version: 5.5.56-MariaDB MariaDB Server

Copyright (c) 2000, 2017, Oracle, MariaDB Corporation Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> create database hive;
Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database hue;
Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database actm;
Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database rptm;
Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database oozie;
Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> exit
Bye

Each Node Side Configuration

Nodes command history
    1  exit
    2  vi /etc/hosts
    3  ssh-keygen -t rsa
    4  clear
    5  ssh-copy-id root@master
    6  clear
    7  ssh-copy-id root@node1
    8  ssh-copy-id root@node2
    9  clear
   10  vi /etc/selinux/config
   11  systemctl stop firewalld
   12  systemctl disable firewalld
   13  systemctl disable iptables
   14  systemctl disable ip6tables
   15  clear
   16  vi /etc/default/grub
   17  clear
   18  yum -y install ntp
   19  systemctl start ntpd
   20  systemctl enable ntpd
   21  clear
   22  reboot
   23  vi /etc/yum.repos.d/cloudera.repo
   24  yum install cloudera-manager-agent.x86_64 cloudera-manager-daemons.x86_64 oracle-j2sdk1.7.x86_64 enterprise-debuginfo.x86_64
   25  clear
   26  vi /etc/default/cloudera-scm-agent
   27  vi /etc/cloudera-scm-agent/config.ini
   28  service cloudera-scm-agent start
   29  chkconfig cloudera-scm-agent on
   30  clear
   31  echo never > /sys/kernel/mm/transparent_hugepage/enabled
   32  echo never > /sys/kernel/mm/transparent_hugepage/defrag
   33  vi /etc/sysctl.conf