20 July 2015

Step by Step Oracle Database 12c Release 1 (12.1.0.2) RAC On Oracle Linux 5.6 X64

Oracle Database 12c Release 1 (12.1.0.2) RAC On Oracle Linux 5.6 x86_64

This article describes the installation of Oracle Database 12c release 1 (12.1.0.2 64-bit) RAC on Linux (Oracle Linux 5.6 64-bit)
RAC and Oracle Clusterware Best Practices and Starter Kit (Platform Independent) (Doc ID 810394.1)
Steps To Shutdown(stop)/Startup(start) CRS, OHAS, ASM, RDBMS & ACFS Services on a RAC Cluster 11.2 & 12.1 Configuration (Doc ID 1355977.1)

How To Setup ASM (11.2 & 12.1) On An Active/Passive Cluster (Non-RAC). (Doc ID 1296124.1)

Grid Infrastructure and RAC Installation, Upgrade, and Patching (Doc ID 1998696.1)

###################Node1 #############################
[oracle@rac1 .ssh]$ cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.
127.0.0.1       localhost.localdomain localhost
::1             localhost6.localdomain6 localhost6

############### Public IP ADDRESS#########
192.168.110.128 rac1.test.com   rac1
192.168.110.130 rac2.test.com   rac2
#############Prive IP ADDRESS#############
192.168.88.15 rac1-priv.test.com      rac1-priv
192.168.88.16 rac2-priv.test.com      rac2-priv
###############VIP #######################
192.168.110.131  rac2-vip.test.com       rac2-vip
192.168.110.129 rac1-vip.test.com       rac1-vip
192.168.88.17   openfiler.test.com      openfiler
192.168.110.132                rac-scan
192.168.110.133                rac-scan
192.168.110.134                rac-scan
#########################NODE-2#######################
[oracle@rac2 .ssh]$ cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.
127.0.0.1       localhost.localdomain localhost
::1             localhost6.localdomain6 localhost6
############### Public IP ADDRESS#########
192.168.110.128 rac1.test.com   rac1
192.168.110.130 rac2.test.com   rac2
#############Prive IP ADDRESS#############
192.168.88.15 rac1-priv.test.com      rac1-priv
192.168.88.16 rac2-priv.test.com      rac2-priv
###############VIP #######################
192.168.110.131  rac2-vip.test.com       rac2-vip
192.168.110.129 rac1-vip.test.com       rac1-vip
192.168.88.17   openfiler.test.com      openfiler
192.168.110.132                rac-scan
192.168.110.133                rac-scan
192.168.110.134                rac-scan

##############################NODE -1 ##################
[root@rac1 iscsi]# mkdir -p /u01/app/12.1.0.2/grid
 [root@rac1 iscsi]# mkdir -p /u01/app/oracle/product/12.1.0.2/db_1
 [root@rac1 iscsi]# chown -R oracle:dba /u01
 [root@rac1 iscsi]# chmod -R 775 /u01/app/oracle
[root@rac1 iscsi]# chmod -R 775 /u01
Add following line for user oracle to set shell limits in file /etc/security/limits.conf
[root@rac1 iscsi]# vi /etc/security/limits.conf
# shell limits for users oracle 12.1.0.2
oracle   soft   nofile   1024
oracle   hard   nofile   65536
oracle   soft   nproc    2047
oracle   hard   nproc    16384
oracle   soft   stack    10240
oracle   hard   stack    32768

########################NODE-2 #########################
Create the directories in which the Oracle software will be installed.

[root@rac2 ~]# mkdir -p  /u01/app/12.1.0.2/grid
[root@rac2 ~]# mkdir -p /u01/app/oracle/product/12.1.0.2/db_1
[root@rac2 ~]# chown -R oracle:oinstall /u01
[root@rac2 ~]# chmod -R 775 /u01/
Add following line for user oracle to set shell limits in file /etc/security/limits.conf
[root@rac2]# vi /etc/security/limits.conf
# shell limits for users oracle 12.1.0.2

oracle   soft   nofile   1024
oracle   hard   nofile   65536
oracle   soft   nproc    2047
oracle   hard   nproc    16384
oracle   soft   stack    10240
oracle   hard   stack    32768

Oracle Installation Prerequisites

yum install oracle-rdbms-server-12cR1-preinstall -y

Manual Setup

If you have not installed oracle-rdbms-server-12cR1-preinstall package, you need to manually perform the following setup tasks.
########################NODE-1 #########################
Add or amend the following lines to the "/etc/sysctl.conf" file.

fs.file-max = 6815744
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
kernel.shmall = 1073741824
kernel.shmmax = 4398046511104
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range = 9000 65500

Apply kernel parameters
 [root@rac1 ~]# sysctl -p


Create the new groups and users.

[root@rac1 ~]# groupadd -g 54321 oinstall
[root@rac1 ~]# groupadd -g 54322 dba
[root@rac1 ~]# groupadd -g 54323 oper
[root@rac1 ~]# #groupadd -g 54324 backupdba
[root@rac1 ~]# #groupadd -g 54325 dgdba
[root@rac1 ~]# #groupadd -g 54326 kmdba
[root@rac1 ~]# #groupadd -g 54327 asmdba
[root@rac1 ~]# #groupadd -g 54328 asmoper
[root@rac1 ~]# #groupadd -g 54329 asmadmin

########################NODE-2 #########################
Add or amend the following lines to the "/etc/sysctl.conf" file.

fs.file-max = 6815744
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
kernel.shmall = 1073741824
kernel.shmmax = 4398046511104
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range = 9000 65500

Apply kernel parameters
[root@rac2 ~]# sysctl -p

Create the new groups and users.

[root@rac2 ~]# groupadd -g 54321 oinstall
[root@rac2 ~]# groupadd -g 54322 dba
[root@rac2 ~]# groupadd -g 54323 oper
[root@rac2 ~]# #groupadd -g 54324 backupdba
[root@rac2 ~]# #groupadd -g 54325 dgdba
[root@rac2 ~]# #groupadd -g 54326 kmdba
[root@rac2 ~]# #groupadd -g 54327 asmdba
[root@rac2 ~]# #groupadd -g 54328 asmoper
[root@rac1 ~]# #groupadd -g 54329 asmadmin

########################NODE-1 #########################
To configure ASMlib first you need to download ASMLib rpms

[root@rac1 oracle]# oracleasm configure -i
Configuring the Oracle ASM library driver.

This will configure the on-boot properties of the Oracle ASM library
driver.  The following questions will determine whether the driver is
loaded on boot and what permissions it will have.  The current values
will be shown in brackets ('[]').  Hitting <ENTER> without typing an
answer will keep that current value.  Ctrl-C will abort.

Default user to own the driver interface [oracle]: ----Enter grid user name
Default group to own the driver interface [dba]:  ----Enter grid group name
Start Oracle ASM library driver on boot (y/n) [n]: y
Scan for Oracle ASM disks on boot (y/n) [y]:
Writing Oracle ASM library driver configuration: done

[root@rac1 oracle]# oracleasm init
Creating /dev/oracleasm mount point: /dev/oracleasm
Loading module "oracleasm": oracleasm
Mounting ASMlib driver filesystem: /dev/oracleasm

Create RAW disk:

[root@rac1 oracle]# fdisk /dev/sdb
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel. Changes will remain in memory only,
until you decide to write them. After that, of course, the previous
content won't be recoverable.

The number of cylinders for this disk is set to 57248.
There is nothing wrong with that, but this is larger than 1024,
and could in certain setups cause problems with:
1) software that runs at boot time (e.g., old versions of LILO)
2) booting and partitioning software from other OSs
   (e.g., DOS FDISK, OS/2 FDISK)
Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-57248, default 1):
Using default value 1
Last cylinder or +size or +sizeM or +sizeK (1-57248, default 57248):
Using default value 57248

Command (m for help): wq
The partition table has been altered!

Calling ioctl() to re-read partition table.

WARNING: Re-reading the partition table failed with error 16: Device or resource busy.
The kernel still uses the old table.
The new table will be used at the next reboot.
Syncing disks.
[root@rac1 oracle]# fdisk /dev/sdc
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel. Changes will remain in memory only,
until you decide to write them. After that, of course, the previous
content won't be recoverable.

Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-1009, default 1):
Using default value 1
Last cylinder or +size or +sizeM or +sizeK (1-1009, default 1009):
Using default value 1009

Command (m for help): wq
The partition table has been altered!

Calling ioctl() to re-read partition table.

WARNING: Re-reading the partition table failed with error 16: Device or resource busy.
The kernel still uses the old table.
The new table will be used at the next reboot.
Syncing disks.
[root@rac1 oracle]# fdisk /dev/sdd
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel. Changes will remain in memory only,
until you decide to write them. After that, of course, the previous
content won't be recoverable.

Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-1009, default 1):
Using default value 1
Last cylinder or +size or +sizeM or +sizeK (1-1009, default 1009):
Using default value 1009

Command (m for help): wq
The partition table has been altered!

Calling ioctl() to re-read partition table.

WARNING: Re-reading the partition table failed with error 16: Device or resource busy.
The kernel still uses the old table.
The new table will be used at the next reboot.
Syncing disks.
[root@rac1 oracle]# fdisk -l

Disk /dev/sda: 26.8 GB, 26843545600 bytes
255 heads, 63 sectors/track, 3263 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes

   Device Boot      Start         End      Blocks   Id  System
/dev/sda1   *           1          13      104391   83  Linux
/dev/sda2              14        3002    24009142+  83  Linux
/dev/sda3            3003        3263     2096482+  82  Linux swap / Solaris

Disk /dev/sdb: 60.0 GB, 60028878848 bytes
64 heads, 32 sectors/track, 57248 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes

   Device Boot      Start         End      Blocks   Id  System
/dev/sdb1               1       57248    58621936   83  Linux

Disk /dev/sdc: 2147 MB, 2147483648 bytes
67 heads, 62 sectors/track, 1009 cylinders
Units = cylinders of 4154 * 512 = 2126848 bytes

   Device Boot      Start         End      Blocks   Id  System
/dev/sdc1               1        1009     2095662   83  Linux

Disk /dev/sdd: 2147 MB, 2147483648 bytes
67 heads, 62 sectors/track, 1009 cylinders
Units = cylinders of 4154 * 512 = 2126848 bytes

   Device Boot      Start         End      Blocks   Id  System
/dev/sdd1               1        1009     2095662   83  Linux
[root@rac1 oracle]# oracleasm createdisk DISK1 /dev/sdb1
Writing disk header: done
Instantiating disk: done
[root@rac1 oracle]# oracleasm createdisk DISK2 /dev/sdc
sdc   sdc1
[root@rac1 oracle]# oracleasm createdisk DISK2 /dev/sdc
sdc   sdc1
[root@rac1 oracle]# oracleasm createdisk DISK2 /dev/sdc1
Writing disk header: done
Instantiating disk: done
[root@rac1 oracle]# oracleasm createdisk DISK3 /dev/sdd1
Writing disk header: done
Instantiating disk: done
[root@rac1 oracle]# oracleasm listdisks
DISK1
DISK2
DISK3
########################NODE-2 #########################
[root@rac2 ]# oracleasm listdisks
DISK1
DISK2
DISK3

########################NODE-1 #########################

Configure dnsmasq for SCAN  ----  Node 1 Configuration

create new file “/etc/racdns” with settings for SCAN

192.168.110.132 rac-scan.test.com       rac-scan
192.168.110.133 rac-scan.test.com       rac-scan
192.168.110.134 rac-scan.test.com       rac-scan

[root@rac1 oracle]# service named stop
[root@rac1 oracle]# cat /etc/dnsmasq.conf | grep addn-hosts
#addn-hosts=/etc/banner_add_hosts
[root@rac1 oracle]# vi /etc/dnsmasq.conf  -   Add below parameter
addn-hosts=/etc/racdns
[root@rac1 oracle]# service dnsmasq start
Starting dnsmasq:                                          [  OK  ]
[root@rac1 oracle]# chkconfig named off
[root@rac1 oracle]# chkconfig dnsmasq on

As default dnsmasq is running on ip adress 127.0.0.1 so it requires following settings in “/etc/resolv.conf”.

[root@rac1 oracle]# cat /etc/resolv.conf
; generated by /sbin/dhclient-script
search test.com
nameserver 192.168.1.1
nameserver 127.0.0.1

Verification of DNS

[root@rac1 oracle]# nslookup rac-scan
Server:         127.0.0.1
Address:        127.0.0.1#53

Name:   rac-scan.test.com
Address: 192.168.110.132
Name:   rac-scan.test.com
Address: 192.168.110.133
Name:   rac-scan.test.com
Address: 192.168.110.134

Repeat the same DNS setup for node 2 also

[root@rac2 ~]# nslookup rac-scan
Server:         127.0.0.1
Address:        127.0.0.1#53

Name:   rac-scan.test.com
Address: 192.168.110.134
Name:   rac-scan.test.com
Address: 192.168.110.132
Name:   rac-scan.test.com
Address: 192.168.110.133

Verify network between nodes
########################NODE-1 #########################
[root@rac1 oracle]# hostname
rac1.test.com
You have new mail in /var/spool/mail/root
[root@rac1 oracle]# ping rac1 -c 1
PING rac1.test.com (192.168.110.128) 56(84) bytes of data.
64 bytes from rac1.test.com (192.168.110.128): icmp_seq=1 ttl=64 time=0.044 ms

--- rac1.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.044/0.044/0.044/0.000 ms
[root@rac1 oracle]# ping rac2 -c 1
PING rac2.test.com (192.168.110.130) 56(84) bytes of data.
64 bytes from rac2.test.com (192.168.110.130): icmp_seq=1 ttl=64 time=0.955 ms

--- rac2.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.955/0.955/0.955/0.000 ms
[root@rac1 oracle]# ping rac1-priv -c 1
PING rac1-priv.test.com (192.168.88.15) 56(84) bytes of data.
64 bytes from rac1-priv.test.com (192.168.88.15): icmp_seq=1 ttl=64 time=0.042 ms

--- rac1-priv.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.042/0.042/0.042/0.000 ms
[root@rac1 oracle]# ping rac2-priv -c 1
PING rac2-priv.test.com (192.168.88.16) 56(84) bytes of data.
64 bytes from rac2-priv.test.com (192.168.88.16): icmp_seq=1 ttl=64 time=0.268 ms

--- rac2-priv.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.268/0.268/0.268/0.000 ms
[root@rac1 oracle]#
########################NODE-2 #########################

[root@rac2 ~]# hostname
rac2.test.com
You have new mail in /var/spool/mail/root
[root@rac2 ~]# ping rac1 -c 1
PING rac1.test.com (192.168.110.128) 56(84) bytes of data.
64 bytes from rac1.test.com (192.168.110.128): icmp_seq=1 ttl=64 time=1.98 ms

--- rac1.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.981/1.981/1.981/0.000 ms
[root@rac2 ~]# ping rac2 -c 1
PING rac2.test.com (192.168.110.130) 56(84) bytes of data.
64 bytes from rac2.test.com (192.168.110.130): icmp_seq=1 ttl=64 time=0.022 ms

--- rac2.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.022/0.022/0.022/0.000 ms
[root@rac2 ~]# ping rac1-priv -c 1
PING rac1-priv.test.com (192.168.88.15) 56(84) bytes of data.
64 bytes from rac1-priv.test.com (192.168.88.15): icmp_seq=1 ttl=64 time=2.89 ms

--- rac1-priv.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.899/2.899/2.899/0.000 ms
[root@rac2 ~]# ping rac2-priv -c 1
PING rac2-priv.test.com (192.168.88.16) 56(84) bytes of data.
64 bytes from rac2-priv.test.com (192.168.88.16): icmp_seq=1 ttl=64 time=0.019 ms

--- rac2-priv.test.com ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.019/0.019/0.019/0.000 ms
[root@rac2 ~]#

Oracle is using for automatic memory management Linux shared segemnts. Usually they are too small but you can modify it on-the-fly. For this presentation we need 2500M.
Just modify entry in “/etc/fstab” to have persistent settings between reboot of your machine
tmpfs   /dev/shm   tmpfs  default,size=2G  0 0
remount it
mount -o remount tmpfs
 
Repeat the steps for Node -2
 
Change NTP
NTP modify “/etc/sysconfig/ntpd” file to
OPTIONS="-x -u ntp:ntp -p /var/run/ntpd.pid"
and restart NTP
# service ntpd restart
 
Repeat the steps for Node -2 also

Start grid software installation as user oracle, remember to run command xhost+ as root.

Install package cvuqdisk-1.0.9-1.rpm from grid software as user root


  
 






 








Check Cluster status on Node 1 and Node 2

[root@rac1 ~]# crsctl stat res -t
--------------------------------------------------------------------------------
Name           Target  State        Server                   State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.PROD.dg
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.asm
               ONLINE  ONLINE       rac1                     Started,STABLE
               ONLINE  ONLINE       rac2                     Started,STABLE
ora.net1.network
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
ora.ons
               ONLINE  ONLINE       rac1                     STABLE
               ONLINE  ONLINE       rac2                     STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       rac1                     STABLE
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       rac1                     STABLE
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       rac1                     STABLE
ora.MGMTLSNR
      1        OFFLINE OFFLINE                               STABLE
ora.cvu
      1        ONLINE  ONLINE       rac1                     STABLE
ora.oc4j
      1        ONLINE  ONLINE       rac1                     STABLE
ora.prod.db
      1        ONLINE  ONLINE       rac1                     Open,STABLE
      2        ONLINE  ONLINE       rac2                     Open,STABLE
ora.rac1.vip
      1        ONLINE  ONLINE       rac1                     STABLE
ora.rac2.vip
      1        ONLINE  ONLINE       rac2                     STABLE
ora.scan1.vip
      1        ONLINE  ONLINE       rac1                     STABLE
ora.scan2.vip
      1        ONLINE  ONLINE       rac1                     STABLE
ora.scan3.vip
      1        ONLINE  ONLINE       rac1                     STABLE

Install 12.1.0.2 oracle database:





















 






















 







 






[root@rac1 ~]#sqlplus "/as sysdba"

SQL*Plus: Release 12.1.0.2.0 Production on Sat Jul 18 17:40:30 2015

Copyright (c) 1982, 2014, Oracle.  All rights reserved.

Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Advanced Analytics and Real Application Testing options

SQL> select INSTANCE_NUMBER,INSTANCE_NAME,HOST_NAME,VERSION from gv$instance;

INSTANCE_NUMBER INSTANCE_NAME    HOST_NAME            VERSION
--------------- ---------------- -------------------- -----------------
              1 prod1            rac1.test.com        12.1.0.2.0
              2 prod2            rac2.test.com        12.1.0.2.0
 




No comments: