Oracle11g_RAC_OELR5-U4_iSCSI（非域名解析）两节点搭建

ClusterWare软件(Grid)   本地文件系统        本地磁盘
    voting disk(Grid)       ASM                 共享磁盘
    OCR(Grid)               ASM                 共享磁盘
    数据库软件              本地文件系统        本地磁盘
    数据库                  ASM                 共享磁盘

# rac1 rac2 执行

vi /etc/hosts
#public
192.168.100.101   rac1
192.168.100.102   rac2
#vip
192.168.100.11   rac1-vip
192.168.100.12   rac2-vip
#private
100.100.0.101 rac1-priv
100.100.0.102 rac2-priv
#scan
192.168.100.103 rac-scan


wget http://public-yum.oracle.com/public-yum-ol6.repo

yum install binutils-2.* compat-libstdc++-33* elfutils-libelf-0.* elfutils-libelf-devel-* gcc-4.* gcc-c++-4.* glibc-2.* glibc-common-2.* glibc-devel-2.* glibc-headers-2.* ksh-2* libaio-0.* libaio-devel-0.* libgcc-4.* libstdc++-4.* libstdc++-devel-4.* make-3.* sysstat-7.* unixODBC-2.* unixODBC-devel-2.* pdksh*



rpm -q binutils compat-libstdc++-33 elfutils-libelf elfutils-libelf-devel gcc gcc-c++
rpm -q glibc glibc-common glibc-devel glibc-headers ksh libaio libaio-devel
rpm -q libgcc libstdc++ libstdc++-devel make numactl-devel sysstat unixODBC unixODBC-devel

mount /dev/cdrom /mnt/hgfs/
rpm -ivh /mnt/hgfs/Server/Packages/numactl-devel-2.0.7-6.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-db42-4.2.52-15.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-db43-4.3.29-15.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-db-4.6.21-15.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/compat-libcap1-1.10-1.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/unixODBC-devel-2.2.14-12.el6_3.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/libstdc++-devel-4.4.7-3.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/libaio-devel-0.3.107-10.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/kernel-headers-2.6.32-358.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/glibc-headers-2.12-1.107.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/glibc-devel-2.12-1.107.el6.x86_64.rpm
rpm -ivh /mnt/hgfs/Server/Packages/ksh-20100621-19.el6.x86_64.rpm

cat >> /etc/security/limits.conf <<EOF
oracle              soft    nproc   2047
oracle              hard    nproc   16384
oracle              soft    nofile  1024
oracle              hard    nofile  65536
oracle              soft    stack   10240
grid                soft    nproc   2047
grid                hard    nproc   16384
grid                soft    nofile  1024
grid                hard    nofile  65536
grid                soft    stack   10240
EOF

vi /etc/selinux/config
service iptables save
service iptables stop
/etc/rc.d/init.d/iptables status




#禁用 NTP  采用 CTSS
/sbin/service ntpd stop
chkconfig ntpd off
mv /etc/ntp.conf /etc/ntp.conf.original                       
rm /var/run/ntpd.pid       

rpm -ivh /mnt/hgfs/Server/Packages/iscsi-initiator-utils-*     

service iscsi start
service iscsid start
chkconfig iscsid on
chkconfig iscsi on
          
service iscsid start
chkconfig --list | grep iscsi
cat messages | grep iscsi


iscsiadm -m discovery -t sendtargets -p 100.100.0.43

100.100.0.43:3260,1 iqn.2006-01.com.openfiler:racdb.fra1
100.100.0.43:3260,1 iqn.2006-01.com.openfiler:racdb.data1
100.100.0.43:3260,1 iqn.2006-01.com.openfiler:racdb.crs1

iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.crs1 -p 100.100.0.43 -l
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.data1 -p 100.100.0.43 -l
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.fra1 -p 100.100.0.43 -l
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.crs1 -p 100.100.0.43 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.data1 -p 100.100.0.43 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:racdb.fra1 -p 100.100.0.43 --op update -n node.startup -v automatic


(cd /dev/disk/by-path; ls -l *openfiler* | awk '{FS=" "; print $9 " " $10 " " $11}')

touch /etc/udev/rules.d/99-asm-iscsi.rules
cat >>/etc/udev/rules.d/99-asm-iscsi.rules <<EOF
SUBSYSTEM=="block",KERNEL=="sd*",ENV{ID_SERIAL}=="14f504e46494c45524f4452576c352d4d58386e2d4a7a6a53", BUS=="scsi",SYMLINK+="iscsi/crs1/part"
SUBSYSTEM=="block",KERNEL=="sd*",ENV{ID_SERIAL}=="14f504e46494c4552467248686c4c2d693570742d49314d6f", BUS=="scsi",SYMLINK+="iscsi/data1/part"
SUBSYSTEM=="block",KERNEL=="sd*",ENV{ID_SERIAL}=="14f504e46494c455255476461584a2d734a31382d63724845", BUS=="scsi",SYMLINK+="iscsi/fra1/part"
EOF


udevadm trigger
start_udev

ls -l /dev/iscsi/*



service iscsi stop
service iscsi start

dd if=/dev/zero of=/dev/iscsi/crs1/part bs=1024 count=100
dd if=/dev/zero of=/dev/iscsi/data1/part bs=1024 count=100
dd if=/dev/zero of=/dev/iscsi/fra1/part bs=1024 count=100


**********************************************************************************************0
su - root
userdel grid
rm -rf /home/grid

userdel oracle
rm -rf /home/oracle

groupdel oinstall
groupdel dba
groupdel oper  
groupdel asmadmin
groupdel asmdba
groupdel asmoper

groupadd -g 500 oinstall
groupadd -g 501 dba
groupadd -g 502 oper  
groupadd -g 503 asmadmin
groupadd -g 504 asmdba
groupadd -g 505 asmoper


useradd -m -u 500 -g oinstall -G asmadmin,asmdba,asmoper -d /home/grid -s /bin/bash -c "Grid Infrastructure Owner" grid
passwd grid


useradd -m -u 501 -g oinstall -G dba,oper,asmdba -d /home/oracle -s /bin/bash -c "Oracle Software Owner" oracle
passwd oracle


fdisk /dev/sdc
mkfs -t ext4 /dev/sdc1
cat >> /etc/fstab <<EOF
/dev/sdc1               /u01                    ext4    defaults        1 2
EOF

rm -fr /u01
mkdir /u01
mount /dev/sdb1 /u01

mkdir -p /u01/app/grid
mkdir -p /u01/app/oracle
mkdir -p /u01/app/11.2.0/grid
mkdir -p /u01/app/oraInventory

chown -R grid:oinstall /home/grid
chown -R grid:asmadmin /dev/iscsi/*
chown -R grid:oinstall /u01/app/grid
chown -R grid:oinstall /u01/app/11.2.0/grid
chown -R oracle:oinstall /u01/app/oracle
chown -R grid:oinstall  /u01/app/oraInventory

chmod -R 775 /u01/app/grid/
chmod -R 775 /u01/app/oracle/
chmod -R 775 /u01/app/11.2.0/grid
chmod -R 775 /u01/app/oraInventory



id grid
###########################
su - grid
cat /dev/null > .bash_profile
cat > .bash_profile <<EOF
# ---------------------------------------------------
# .bash_profile
# ---------------------------------------------------
# OS User:      grid
# Application:  Oracle Grid Infrastructure
# Version:      Oracle 11g release 2
# ---------------------------------------------------

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
    . ~/.bashrc
fi
alias ls="ls -FA"

ORACLE_SID=+ASM3; export ORACLE_SID

JAVA_HOME=/usr/local/java; export JAVA_HOME

ORACLE_BASE=/u01/app/grid; export ORACLE_BASE

ORACLE_HOME=/u01/app/11.2.0/grid; export ORACLE_HOME

ORACLE_PATH=/u01/app/oracle/common/oracle/sql; export ORACLE_PATH

ORACLE_TERM=xterm; export ORACLE_TERM

NLS_DATE_FORMAT="YYYY-MON-DD HH24:MI:SS"; export NLS_DATE_FORMAT

TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN

ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11

PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH

THREADS_FLAG=native; export THREADS_FLAG

export TEMP=/tmp
export TMPDIR=/tmp
export TMP=/tmp
export CVUQDISK_GRP=oinstall

export LANG=en_US.UTF-8

umask 022
EOF
-------------------

su - oracle

cat /dev/null > .bash_profile
cat > .bash_profile <<EOF
# ---------------------------------------------------
# .bash_profile
# ---------------------------------------------------
# OS User:      oracle
# Application:  Oracle Database Software Owner
# Version:      Oracle 11g release 2
# ---------------------------------------------------

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
      . ~/.bashrc
fi

alias ls="ls -FA"

ORACLE_SID=racdb3; export ORACLE_SID

ORACLE_UNQNAME=racdb; export ORACLE_UNQNAME

JAVA_HOME=/usr/local/java; export JAVA_HOME

ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE

ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1; export ORACLE_HOME

ORACLE_PATH=/u01/app/common/oracle/sql; export ORACLE_PATH

ORACLE_TERM=xterm; export ORACLE_TERM

NLS_DATE_FORMAT="YYYY-MON-DD HH24:MI:SS"; export NLS_DATE_FORMAT

TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN

ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11

PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH

THREADS_FLAG=native; export THREADS_FLAG

export TEMP=/tmp
export TMPDIR=/tmp

export LANG=en_US.UTF-8

umask 022
EOF
----------------------
su - root

cat >>/etc/profile <<EOF
if [ \$USER = "oracle" ] || [ \$USER = "grid" ]; then
    if [ \$SHELL = "/bin/ksh" ]; then
        ulimit -p 16384
        ulimit -n 65536
    else
        ulimit -u 16384 -n 65536
    fi
    umask 022
fi
EOF

cat >> /etc/csh.login <<EOF
if ( \$USER == "oracle" || \$USER == "grid" ) then
    limit maxproc 16384
    limit descriptors 65536
endif
EOF

cat /proc/meminfo | grep MemTotal
cat /proc/meminfo | grep SwapTotal

dd if=/dev/zero of=tempswap bs=1k count=4000000
chmod 600 tempswap
mke2fs tempswap <<EOF
y
EOF

mkswap tempswap
swapon tempswap

cat /proc/meminfo | grep SwapTotal
*******************************************************************************************1

rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n"| grep ssh
#openssh-clients-5.3p1-84.1.el6 (x86_64)
#libssh2-1.4.2-1.el6 (x86_64)
#openssh-5.3p1-84.1.el6 (x86_64)
#openssh-server-5.3p1-84.1.el6 (x86_64)
#openssh-askpass-5.3p1-84.1.el6 (x86_64)

-------------------------------
#grid  oracle 各执行一次

方法一(推荐)：
#（替代方法）利用OUI 界面 cluster node -> setup 步骤安装, 或静默方式
(
su - grid
cd /media/soft/gridsoft/grid/sshsetup
./sshUserSetup.sh -user grid -hosts "rac1 rac2" -advanced -exverify

su - oracle
cd /media/soft/gridsoft/grid/sshsetup
./sshUserSetup.sh -user oracle -hosts "rac1 rac2" -advanced -exverify

)

方法二：
#RAC1 rac2  grid  oracle 各执行一次
[grid@rac1 ~]$ su - grid
[grid@rac1 ~]$ mkdir ~/.ssh
[grid@rac1 ~]$ chmod 700 ~/.ssh
[grid@rac1 ~]$ /usr/bin/ssh-keygen -t dsa
[grid@rac1 ~]$ /usr/bin/ssh-keygen -t rsa
[grid@rac1 ~]$ touch ~/.ssh/authorized_keys
[grid@rac1 ~]$ ls -l ~/.ssh
[grid@rac1 ~]$ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
[grid@rac1 ~]$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

[grid@rac2 ~]$ su - grid
[grid@rac2 ~]$ mkdir ~/.ssh
[grid@rac2 ~]$ chmod 700 ~/.ssh
[grid@rac2 ~]$ /usr/bin/ssh-keygen -t dsa
[grid@rac2 ~]$ /usr/bin/ssh-keygen -t rsa
[grid@rac2 ~]$ touch ~/.ssh/authorized_keys
[grid@rac2 ~]$ ls -l ~/.ssh
[grid@rac2 ~]$ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
[grid@rac2 ~]$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

[grid@rac1 ~]$ ssh rac2 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
[grid@rac1 ~]$ ssh rac2 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
[grid@rac1 ~]$ ls -l ~/.ssh
[grid@rac1 ~]$ scp ~/.ssh/authorized_keys rac2:.ssh/authorized_keys
[grid@rac1 ~]$ chmod 600 ~/.ssh/authorized_keys

[grid@rac2 ~]$ chmod 600 ~/.ssh/authorized_keys

[grid@rac2 ~]$ ssh rac1 "date;hostname"
[grid@rac1 ~]$ ssh rac2 "date;hostname"
[grid@rac2 ~]$ ssh rac1 hostname
[grid@rac1 ~]$ ssh rac2 hostname
------------------------
 cat /etc/sysctl.conf

cat >> /etc/sysctl.conf <<EOF

# Controls the maximum number of shared memory segments system wide
kernel.shmmni = 4096

# Sets the following semaphore values:
# SEMMSL_value  SEMMNS_value  SEMOPM_value  SEMMNI_value
kernel.sem = 250 32000 100 128

# Sets the maximum number of file-handles that the Linux kernel will allocate
fs.file-max = 6815744

# Defines the local port range that is used by TCP and UDP
# traffic to choose the local port
net.ipv4.ip_local_port_range = 9000 65500

# Default setting in bytes of the socket "receive" buffer which
# may be set by using the SO_RCVBUF socket option
net.core.rmem_default=262144

# Maximum setting in bytes of the socket "receive" buffer which
# may be set by using the SO_RCVBUF socket option
net.core.rmem_max=4194304

# Default setting in bytes of the socket "send" buffer which
# may be set by using the SO_SNDBUF socket option
net.core.wmem_default=262144

# Maximum setting in bytes of the socket "send" buffer which
# may be set by using the SO_SNDBUF socket option
net.core.wmem_max=1048576

# Maximum number of allowable concurrent asynchronous I/O requests requests
fs.aio-max-nr=1048576
EOF

sysctl -p

------------------------
rpm -ivh /media/soft/oracleasmlib-2.0.4-1.el6.x86_64.rpm
rpm -ivh /media/soft/oracleasm-support-2.1.8-1.el6.x86_64.rpm

scp /media/soft/oracleasmlib-2.0.4-1.el6.x86_64.rpm  rac2:/home/grid/oracleasmlib-2.0.4-1.el6.x86_64.rpm

scp /media/soft/oracleasm-support-2.1.8-1.el6.x86_64.rpm  rac2:/home/grid/oracleasm-support-2.1.8-1.el6.x86_64.rpm

[root@rac2 ~]# rpm -ivh /home/grid/oracleasmlib-2.0.4-1.el6.x86_64.rpm

[root@rac2 ~]# rpm -ivh /home/grid/oracleasm-support-2.1.8-1.el6.x86_64.rpm

------rac1   rac2 ----------------
/usr/sbin/oracleasm configure -i <<EOF
grid
asmadmin
y
y
EOF
/usr/sbin/oracleasm init
------------------------
为了防止selinux阻止访问disk header
备份/etc/selinux/config
# cp /etc/selinux/config  /etc/selinux/config_bak

b)修改/etc/selinux/config
注释掉   SELINUX=enforcing
添加     SELINUX=disabled
vi /etc/selinux/config
# SELINUX=enforcing
  SELINUX=disabled
c)重启操作系统
# reboot

su - root
fdisk /dev/sdd<<EOF
n
p
1
1

w
EOF


fdisk /dev/sde<<EOF
n
p
1
1

w
EOF

fdisk /dev/sdf<<EOF
n
p
1
1

w
EOF

----rac1 rac2----
start_udev

/usr/sbin/oracleasm createdisk CRSVOL1 /dev/iscsi/crs1/part

/usr/sbin/oracleasm createdisk DATAVOL1 /dev/iscsi/data1/part

/usr/sbin/oracleasm createdisk FRAVOL1 /dev/iscsi/fra1/part


/usr/sbin/oracleasm scandisks
/usr/sbin/oracleasm listdisks

mount /dev/cdrom /mnt/hgfs

rpm -ivh /media/soft/gridsoft/grid/rpm/cvuqdisk-1.0.9-1.rpm

scp /media/soft/gridsoft/grid/rpm/cvuqdisk-1.0.9-1.rpm rac2:/home/grid/cvuqdisk-1.0.9-1.rpm

ssh rac2 rpm -ivh /home/grid/cvuqdisk-1.0.9-1.rpm


*********************************************************************
-----------------------------------
(注意：配置 DNS无需此步骤)
--------rac1----------
mv /usr/bin/nslookup /usr/bin/nslookup.original

cat > /usr/bin/nslookup <<EOF
#!/bin/bash

HOSTNAME=${1}

if [[ $HOSTNAME = "rac-scan" ]]; then
    echo "Server:          24.154.1.34"
    echo "Address:         24.154.1.34#53"
    echo "Non-authoritative answer:"
    echo "Name:    rac-scan"
    echo "Address:  192.168.100.90"
else
    /usr/bin/nslookup.original $HOSTNAME
fi
EOF

chmod 755 /usr/bin/nslookup

--------rac2----------
mv /usr/bin/nslookup /usr/bin/nslookup.original

cat > /usr/bin/nslookup <<EOF
#!/bin/bash

HOSTNAME=${1}

if [[ $HOSTNAME = "rac-scan" ]]; then
    echo "Server:          24.154.1.34"
    echo "Address:         24.154.1.34#53"
    echo "Non-authoritative answer:"
    echo "Name:    rac-scan"
    echo "Address:  192.168.100.90"
else
    /usr/bin/nslookup.original $HOSTNAME
fi
EOF

chmod 755 /usr/bin/nslookup
------------------------------------20170412
su - grid
cd /media/soft/gridsoft/grid/
./runcluvfy.sh stage -pre crsinst -n rac1,rac2 -fixup -verbose

cluvfy.sh stage -pre crsinst -n rac1,rac2 -fixup -verbose

---------------------------
su - root
xhost +
su - grid
cd /media/soft/gridsoft/grid/
./runInstaller

------------------------
cd /media/soft/gridsoft/grid/
./runcluvfy.sh comp scan -verbose

cluvfy comp scan -verbose

-------------------------------------------

$ORACLE_HOME/bin/crsctl check crs

$ORACLE_HOME/bin/olsnodes -n

ps -ef | grep lsnr | grep -v 'grep' | grep -v 'ocfs' | awk '{print $9}'

$ORACLE_HOME/bin/srvctl status asm -a

$ORACLE_HOME/bin/ocrcheck

$ORACLE_HOME/bin/crsctl query css votedisk

asmca
(create diskgroup:  +DATA +FRA)
------------------------------------------
su - oracle
cd /media/soft/oraclesoft/database
./runInstaller
(install soft only)
dbca


ASMCMD [+fra] > ls
ASMCMD [+fra] > mkdir archive
ASMCMD [+fra] > ls
archive/
ASMCMD [+fra] > cd archive
ASMCMD [+fra/archive] > mkdir racdb
ASMCMD [+fra/archive] >

------------------
Service-Side TAF

[oracle@rac1 ~]$ srvctl add service -d racdb -s TAF -m BASIC -e SELECT -z 180 -w 5 -j LONG -r "racdb1,racdb2"
srvctl start service -d racdb -s TAF -i racdb1
srvctl start service -d racdb -s TAF -i racdb2

begin
 dbms_service.modify_service(service_name => 'TAF',
aq_ha_notifications => true,
failover_method => dbms_service.failover_method_basic,
failover_type => dbms_service.failover_type_select,
failover_retries => 180,
failover_delay => 5,
clb_goal => dbms_service.clb_goal_long);
end;
/

select name, failover_method,failover_type, failover_retries,goal, clb_goal,aq_ha_notifications
from dba_services where service_id =
(select service_id from dba_services where name = 'TAF');