安装结束后安装一下包：
rpm -Uvh binutils-2.*
rpm -Uvh elfutils-libelf-0.*
rpm -Uvh glibc-2.*
rpm -Uvh glibc-common-2.*
rpm -Uvh libaio-0.*
rpm -Uvh libgcc-4.*
rpm -Uvh libstdc++-4.*
rpm -Uvh make-3.*
rpm -Uvhkernel-headers-2.6.18-274.el5.x86_64.rpm
rpm -Uvh glibc-headers-2.*
rpm -Uvh glibc-devel-2.*
rpm -Uvhelfutils-libelf-0.137-3.el5.i386.rpm
rpm -Uvh elfutils-libelf-devel*
rpm -Uvhelfutils-libelf-devel-0.*
rpm -Uvh gcc-4.*
rpm -Uvhlibstdc++-devel-4.*
rpm -Uvh gcc-c++-4.*
rpm -Uvh unixODBC-2.*
rpm -Uvh compat-libstdc++-296*
rpm -Uvh compat-libstdc++-33*
rpm -Uvh libaio-devel-0.*
rpm -Uvh libXp-1.*
rpm -Uvh unixODBC-devel-2.*
rpm -Uvh sysstat-7.*
1) 在两个节点上编辑/etc/hosts文件：
127.0.0.1 localhost.localdomain localhost
#used by ntp serviceto synchronize time
66.187.233.4 clock.redhat.com
#local network forRAC
192.168.0.131 ora10racn1 ora10racn1.ccz.com
192.168.0.132 ora10racn2 ora10racn2.ccz.com
192.168.0.151 ora10racn1-vip
192.168.0.152 ora10racn2-vip
192.168.2.131 ora10racn1-str
192.168.2.132 ora10racn2-str
10.10.0.131 ora10racn1-priv
10.10.0.132 ora10racn2-priv
192.168.0.110 openfiler
192.168.2.110 openfiler-str
2) 确保节点的主机名称不在loopback中出现：
127.0.0.1 localhost.localdomain localhost
3) 修改/etc/sysctl.conf中的网络设置：
#+---------------------------------------------------------+
# | Default settingin bytes of the socket "receive" buffer |
# | which may be setby using the SO_RCVBUF socket option. |
#+---------------------------------------------------------+
net.core.rmem_default=1048576
#+---------------------------------------------------------+
# | Maximum settingin bytes of the socket "receive" buffer |
# | which may be setby using the SO_RCVBUF socket option. |
#+---------------------------------------------------------+
net.core.rmem_max=1048576
#+---------------------------------------------------------+
# | Default settingin bytes of the socket "send" buffer |
# | which may be setby using the SO_SNDBUF socket option. |
#+---------------------------------------------------------+
net.core.wmem_default=262144
#+---------------------------------------------------------+
# | Maximum settingin bytes of the socket "send" buffer |
# | which may be setby using the SO_SNDBUF socket option. |
#+---------------------------------------------------------+
net.core.wmem_max=262144
4) 关闭所有节点上的UDP ICMP的rejection
[root@ora10racn1Server]# /etc/rc.d/init.d/iptables status
Firewall is stopped.
如果是running状态需要用以下命令关闭：
[root@ora10racn1Server]# /etc/rc.d/init.d/iptables stop
在各个运行级别上关闭iptables
[root@ora10racn1Server]# chkconfig --list|grep iptables
iptables 0:off 1:off 2:on 3:on 4:on 5:on 6:off
[root@ora10racn1Server]# chkconfig iptables off
[root@ora10racn1Server]# chkconfig --list|grep iptables
iptables 0:off 1:off 2:off 3:off 4:off 5:off 6:off
# groupadd -g 501 oinstall
# groupadd -g 502 dba
# groupadd -g 503 oper
# useradd -m -u 501 -g oinstall-G dba,oper -d /home/oracle -s /bin/bash oracle
# id oracle
uid=501(oracle) gid=501(oinstall)groups=501(oinstall),502(dba),503(oper)
# passwd oracle
Changing password for useroracle.
New UNIX password: xxxxxxxxxxx
Retype new UNIX password:xxxxxxxxxxx
passwd: all authentication tokensupdated successfully.
修改oracle帐号的.bash_profile
export JAVA_HOME=/usr/local/java
# User specific environment andstartup programs
exportORACLE_BASE=/u01/app/oracle
exportORACLE_HOME=$ORACLE_BASE/product/10.2.0/db_1
export ORA_CRS_HOME=/u01/app/crs
exportORACLE_PATH=$ORACLE_BASE/dba_scripts/common/sql:.:$ORACLE_HOME/rdbms/admin
export CV_JDKHOME=/usr/local/java
# Each RAC node must have aunique ORACLE_SID. (i.e. racdb1, racdb2,...)
exportORACLE_SID=racdb1
exportPATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin
exportPATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
export PATH=${PATH}:$ORACLE_BASE/dba_scripts/common/bin
export ORACLE_TERM=xterm
exportTNS_ADMIN=$ORACLE_HOME/network/admin
exportORA_NLS10=$ORACLE_HOME/nls/data
exportNLS_DATE_FORMAT="DD-MON-YYYY HH24:MI:SS"
exportLD_LIBRARY_PATH=$ORACLE_HOME/lib
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
exportLD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export CLASSPATH=$ORACLE_HOME/JRE
exportCLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
exportCLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
exportCLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export THREADS_FLAG=native
export TEMP=/tmp
检测nobody帐号：
export TMPDIR=/tmp
# id nobody
uid=99(nobody) gid=99(nobody)groups=99(nobody)
如果nobody不存在，需要增加：
# /usr/sbin/useradd nobody
# chown -Roracle:oinstall /u01/app/oracle
# chmod -R 775/u01/app/oracle
# mkdir -p/u01/app/crs
# chown -Roracle:oinstall /u01/app/crs
# chmod -R 775/u01/app/crs
# mkdir -p /u02
# chown -Roracle:oinstall /u02
# chmod -R 775 /u02
1) 在各个节点安装iscsi的initator：
[root@ora10racn1Server]# rpm -Uvh iscsi-initiator-utils-6.2.0.872-10.0.1.el5.x86_64.rpm
warning:iscsi-initiator-utils-6.2.0.872-10.0.1.el5.x86_64.rpm: Header V3 DSA signature:NOKEY, key ID 1e5e0159
Preparing... ########################################### [100%]
packageiscsi-initiator-utils-6.2.0.872-10.0.1.el5.x86_64 is already installed
2) 在各个节点上启动iscsi的进程：
[root@ora10racn1Server]# service iscsid start
Starting iSCSI daemon:
[ OK ]
3) 在各个节点上搜索openfiler上的相关卷：
[root@ora10racn1Server]# iscsiadm -m discovery -tsendtargets -p 192.168.2.110
192.168.2.110:3260,1iqn.2006-01.com.openfiler:racdb.asm2
192.168.0.110:3260,1iqn.2006-01.com.openfiler:racdb.asm2
192.168.2.110:3260,1iqn.2006-01.com.openfiler:racdb.asm1
192.168.0.110:3260,1iqn.2006-01.com.openfiler:racdb.asm1
192.168.2.110:3260,1iqn.2006-01.com.openfiler:racdb.crs1
192.168.0.110:3260,1iqn.2006-01.com.openfiler:racdb.crs1
4) 在两个节点上手动连接相关卷并配置系统重启时自动连接
[root@ora10racn1 Server]# iscsiadm -m node -Tiqn.2006-01.com.openfiler:racdb.crs1 -p 192.168.2.110 -l
Logging in to [iface: default, target:iqn.2006-01.com.openfiler:racdb.crs1, portal: 192.168.2.110,3260]
Login to [iface: default, target: iqn.2006-01.com.openfiler:racdb.crs1,portal: 192.168.2.110,3260] successful.
[root@ora10racn1 Server]# iscsiadm -m node -Tiqn.2006-01.com.openfiler:racdb.asm1 -p 192.168.2.110 -l
Logging in to [iface: default, target:iqn.2006-01.com.openfiler:racdb.asm1, portal: 192.168.2.110,3260]
Login to [iface: default, target: iqn.2006-01.com.openfiler:racdb.asm1,portal: 192.168.2.110,3260] successful.
[root@ora10racn1 Server]# iscsiadm -m node -Tiqn.2006-01.com.openfiler:racdb.asm2 -p 192.168.2.110 -l
Logging in to [iface: default, target: iqn.2006-01.com.openfiler:racdb.asm2,portal: 192.168.2.110,3260]
Login to [iface: default, target: iqn.2006-01.com.openfiler:racdb.asm2,portal: 192.168.2.110,3260] successful.
[root@ora10racn1 Server]# iscsiadm -m node -Tiqn.2006-01.com.openfiler:racdb.crs1 -p 192.168.2.110 --op update -nnode.startup -v automatic
[root@ora10racn1 Server]# iscsiadm -m node -Tiqn.2006-01.com.openfiler:racdb.asm1 -p 192.168.2.110 --op update -nnode.startup -v automatic
[root@ora10racn1 Server]# iscsiadm -m node -Tiqn.2006-01.com.openfiler:racdb.asm2 -p 192.168.2.110 --op update -nnode.startup -v automatic
5) 在各个节点上建立永久性连接规则，以保证自动连接时设备号不受连接顺序影响：
(cd /dev/disk/by-path; ls -l *openfiler* | awk '{FS=" "; print$9 " " $10 " " $11}')
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.asm1-lun-0-> ../../sdc
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.asm2-lun-0-> ../../sdd
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.crs1-lun-0-> ../../sdb
创建连接规则：
[root@ora10racn1 rules.d]# vi /etc/udev/rules.d/55-openiscsi.rules
KERNEL=="sd*", BUS=="scsi",PROGRAM="/etc/udev/scripts/iscsidev.sh%b",SYMLINK+="iscsi/%c/part%n"
创建连接规则使用的脚本：
[root@ora10racn2 Server]# mkdir -p /etc/udev/scripts
[root@ora10racn2 scripts]# vi /etc/udev/scripts/iscsidev.sh
#!/bin/sh
# FILE: /etc/udev/scripts/iscsidev.sh
BUS=${1}
HOST=${BUS%%:*}
[ -e /sys/class/iscsi_host ] || exit 1
file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/targetname"
target_name=$(cat ${file})
# This is not an open-scsi drive
if [ -z "${target_name}" ]; then
exit 1
fi
# Check if QNAP drive
check_qnap_target_name=${target_name%%:*}
if [ $check_qnap_target_name = "iqn.2004-04.com.qnap" ]; then
target_name=`echo"${target_name%.*}"`
fi
echo "${target_name##*.}"
~
"iscsidev.sh" [New] 25L, 507C written
[root@ora10racn2 scripts]#chmod 755 /etc/udev/scripts/iscsidev.sh
重新启动iscsi服务：
[root@ora10racn2 scripts]# service iscsi stop
Logging out of session [sid: 1, target:iqn.2006-01.com.openfiler:racdb.crs1, portal: 192.168.2.110,3260]
Logging out of session [sid: 2, target:iqn.2006-01.com.openfiler:racdb.asm1, portal: 192.168.2.110,3260]
Logging out of session [sid: 3, target:iqn.2006-01.com.openfiler:racdb.asm2, portal: 192.168.2.110,3260]
Logout of [sid: 1, target: iqn.2006-01.com.openfiler:racdb.crs1, portal:192.168.2.110,3260] successful.
Logout of [sid: 2, target: iqn.2006-01.com.openfiler:racdb.asm1, portal:192.168.2.110,3260] successful.
Logout of [sid: 3, target: iqn.2006-01.com.openfiler:racdb.asm2, portal:192.168.2.110,3260] successful.
Stopping iSCSI daemon:
[root@ora10racn2 scripts]# service iscsi start
iscsid is stopped
[ OK ] iSCSI daemon: [ OK ]
[ OK ]
Setting up iSCSI targets: Logging in to [iface: default, target:iqn.2006-01.com.openfiler:racdb.asm1, portal: 192.168.2.110,3260]
Logging in to [iface: default, target: iqn.2006-01.com.openfiler:racdb.asm1,portal: 192.168.0.110,3260]
Logging in to [iface: default, target:iqn.2006-01.com.openfiler:racdb.asm2, portal: 192.168.2.110,3260]
Logging in to [iface: default, target:iqn.2006-01.com.openfiler:racdb.asm2, portal: 192.168.0.110,3260]
Logging in to [iface: default, target:iqn.2006-01.com.openfiler:racdb.crs1, portal: 192.168.2.110,3260]
Logging in to [iface: default, target:iqn.2006-01.com.openfiler:racdb.crs1, portal: 192.168.0.110,3260]
Login to [iface: default, target: iqn.2006-01.com.openfiler:racdb.asm1,portal: 192.168.2.110,3260] successful.
iscsiadm: Could not login to [iface: default, target:iqn.2006-01.com.openfiler:racdb.asm1, portal: 192.168.0.110,3260].
iscsiadm: initiator reported error (19 - encountered non-retryable iSCSI loginfailure)
Login to [iface: default, target: iqn.2006-01.com.openfiler:racdb.asm2,portal: 192.168.2.110,3260] successful.
iscsiadm: Could not login to [iface: default, target:iqn.2006-01.com.openfiler:racdb.asm2, portal: 192.168.0.110,3260].
iscsiadm: initiator reported error (19 - encountered non-retryable iSCSIlogin failure)
Login to [iface: default, target: iqn.2006-01.com.openfiler:racdb.crs1,portal: 192.168.2.110,3260] successful.
iscsiadm: Could not login to [iface: default, target: iqn.2006-01.com.openfiler:racdb.crs1,portal: 192.168.0.110,3260].
iscsiadm: initiator reported error (19 - encountered non-retryable iSCSIlogin failure)
iscsiadm: Could not log into all portals
[ OK ]
[root@ora10racn2 scripts]# ls -l /dev/iscsi/*
/dev/iscsi/asm1:
total 0
lrwxrwxrwx 1 root root 9 May 16 15:16 part -> ../../sdc
/dev/iscsi/asm2:
total 0
lrwxrwxrwx 1 root root 9 May 16 15:16 part -> ../../sdb
/dev/iscsi/crs1:
total 0
lrwxrwxrwx 1 root root 9 May 16 15:16 part -> ../../sdd
6) 在一个节点上为iSCSI卷创建分区：（意：只能在一个节点上行）
[root@ora10racn1 rules.d]# fdisk /dev/iscsi/crs1/part
Device contains neither a valid DOS partition table, nor Sun, SGI or OSFdisklabel
Building a new DOS disklabel. Changes will remain in memory only,
until you decide to write them. After that, of course, the previous
content won't be recoverable.
Warning: invalid flag 0x0000 of partition table 4 will be corrected byw(rite)
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-1009, default 1): 1
Last cylinder or +size or +sizeM or +sizeK (1-1009, default 1009): 1009
Command (m for help): p
Disk /dev/iscsi/crs1/part: 2147 MB, 2147483648 bytes
67 heads, 62 sectors/track, 1009 cylinders
Units = cylinders of 4154 * 512 = 2126848 bytes
Device Boot Start End Blocks Id System
/dev/iscsi/crs1/part1 1 1009 2095662 83 Linux
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@ora10racn1 rules.d]#fdisk /dev/iscsi/asm1/part
Device contains neither a valid DOS partition table, nor Sun, SGI or OSFdisklabel
Building a new DOS disklabel. Changes will remain in memory only,
until you decide to write them. After that, of course, the previous
content won't be recoverable.
The number of cylinders for this disk is set to 8192.
There is nothing wrong with that, but this is larger than 1024,
and could in certain setups cause problems with:
1) software that runs at boot time (e.g., old versions of LILO)
2) booting and partitioning software from other OSs
(e.g., DOS FDISK, OS/2 FDISK)
Warning: invalid flag 0x0000 of partition table 4 will be corrected byw(rite)
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-8192, default 1): 1
Last cylinder or +size or +sizeM or +sizeK (1-8192, default 8192): 8192
Command (m for help): p
Disk /dev/iscsi/asm1/part: 8589 MB, 8589934592 bytes
64 heads, 32 sectors/track, 8192 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/iscsi/asm1/part1 1 8192 8388592 83 Linux
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
WARNING: Re-reading the partition table failed with error 16: Device orresource busy.
The kernel still uses the old table.
The new table will be used at the next reboot.
Syncing disks.
[root@ora10racn1 rules.d]# fdisk /dev/iscsi/asm2/part
Device contains neither a valid DOS partition table, nor Sun, SGI or OSFdisklabel
Building a new DOS disklabel. Changes will remain in memory only,
until you decide to write them. After that, of course, the previous
content won't be recoverable.
The number of cylinders for this disk is set to 8192.
There is nothing wrong with that, but this is larger than 1024,
and could in certain setups cause problems with:
1) software that runs at boot time (e.g., old versions of LILO)
2) booting and partitioning software from other OSs
(e.g., DOS FDISK, OS/2 FDISK)
Warning: invalid flag 0x0000 of partition table 4 will be corrected byw(rite)
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-8192, default 1): 1
Last cylinder or +size or +sizeM or +sizeK (1-8192, default 8192): 8192
Command (m for help): p
Disk /dev/iscsi/asm2/part: 8589 MB, 8589934592 bytes
64 heads, 32 sectors/track, 8192 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/iscsi/asm2/part1 1 8192 8388592 83 Linux
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
7) 在各个节点上对分区进行检验：
[root@ora10racn1 rules.d]# partprobe
[root@ora10racn1 rules.d]# fdisk -l
Disk /dev/sda: 53.6 GB, 53687091200 bytes
255 heads, 63 sectors/track, 6527 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 13 104391 83 Linux
/dev/sda2 14 6527 52323705 8e Linux LVM
Disk /dev/dm-0: 49.3 GB, 49358569472 bytes
255 heads, 63 sectors/track, 6000 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/dm-0 doesn't contain a valid partition table
Disk /dev/dm-1: 4194 MB, 4194304000 bytes
255 heads, 63 sectors/track, 509 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/dm-1 doesn't contain a valid partition table
Disk /dev/sdb: 8589 MB, 8589934592 bytes
64 heads, 32 sectors/track, 8192 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 8192 8388592 83 Linux
Disk /dev/sdc: 8589 MB, 8589934592 bytes
64 heads, 32 sectors/track, 8192 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdc1 1 8192 8388592 83 Linux
Disk /dev/sdd: 2147 MB, 2147483648 bytes
67 heads, 62 sectors/track, 1009 cylinders
Units = cylinders of 4154 * 512 = 2126848 bytes
Device Boot Start End Blocks Id System
/dev/sdd1 1 1009 2095662 83 Linux
[root@ora10racn1 rules.d]#(cd /dev/disk/by-path; ls -l *openfiler* | awk'{FS=" "; print $9 " " $10 " " $11}')
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.asm1-lun-0-> ../../sdc
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.asm1-lun-0-part1-> ../../sdc1
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.asm2-lun-0-> ../../sdb
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.asm2-lun-0-part1-> ../../sdb1
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.crs1-lun-0-> ../../sdd
ip-192.168.2.110:3260-iscsi-iqn.2006-01.com.openfiler:racdb.crs1-lun-0-part1-> ../../sdd1
[root@ora10racn1 ~]# sysctl -p
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 4294967295
kernel.shmall = 268435456
net.core.rmem_default = 1048576
net.core.rmem_max = 1048576
net.core.wmem_default = 262144
net.core.wmem_max = 262144
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
fs.file-max = 65536
net.ipv4.ip_local_port_range = 1024 65000
[root@ora10racn1 ~]#cat >>/etc/security/limits.conf <<EOF
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
EOF
[root@ora10racn1 ~]#cat >> /etc/pam.d/login<<EOF
session required /lib/security/pam_limits.so
EOF
将下面一段加入到/etc/profile文件的末尾：
if [ \$USER = "oracle"]; then
if [ \$SHELL = "/bin/ksh" ]; then
ulimit -p 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
umask 022
fi
[root@ora10racn1 ~]#find /lib/modules -name"hangcheck-timer.ko"
/lib/modules/2.6.18-274.el5/kernel/drivers/char/hangcheck-timer.ko
/lib/modules/2.6.32-200.13.1.el5uek/kernel/drivers/char/hangcheck-timer.ko
2) 配置hangcheck-timer的两个参数：
[root@ora10racn1 ~]#echo "options hangcheck-timerhangcheck_tick=30 hangcheck_margin=180" >>/etc/modprobe.conf
3) 手动加载hangcheckkernel 模块：
[root@ora10racn1 ~]# echo"/sbin/modprobe hangcheck-timer" >> /etc/rc.local
[root@ora10racn1 ~]#modprobehangcheck-timer
[root@ora10racn1 ~]# grepHangcheck /var/log/messages | tail -2
May 16 17:10:22 ora10racn1 kernel: Hangcheck: startinghangcheck timer 0.9.0 (tick is 60 seconds, margin is 180 seconds).
May 16 17:10:22 ora10racn1 kernel: Hangcheck: Usingget_cycles().
Ø 在节点1上：
[root@ora10racn2 scripts]# su - oracle
[oracle@ora10racn2 ~]$ mkdir -p ~/.ssh
[oracle@ora10racn2 ~]$ chmod 700 ~/.ssh
[oracle@ora10racn2 ~]$ /usr/bin/ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key(/home/oracle/.ssh/id_rsa):
Enter passphrase (empty for nopassphrase):
Enter same passphrase again: /*此处不需要输入密码
Your identification has been saved in/home/oracle/.ssh/id_rsa.
Your public key has been saved in/home/oracle/.ssh/id_rsa.pub.
The key fingerprint is:
47:d4:27:ab:64:82:24:97:d9:85:59:f0:87:49:7d:c7oracle@ora10racn1.ccz.com
[oracle@ora10racn1 ~]$ touch ~/.ssh/authorized_keys
[oracle@ora10racn1 ~]$ cd .ssh
[oracle@ora10racn1 .ssh]$ ls -l *.pub
-rw-r--r-- 1 oracle oinstall 407 May 1620:29 id_rsa.pub
[oracle@ora10racn1 .ssh]$ ssh ora10racn1.ccz.com cat ~/.ssh/id_rsa.pub>>~/.ssh/authorized_keys
The authenticity of host'ora10racn1.ccz.com (192.168.0.131)' can't be established.
RSA key fingerprint isf9:5e:99:da:b1:00:9e:6b:a4:9e:a0:cd:ae:e3:4d:ca.
Are you sure you want to continueconnecting (yes/no)? yes
Warning: Permanently added'ora10racn1.ccz.com,192.168.0.131' (RSA) to the list of known hosts.
oracle@ora10racn1.ccz.com's password: /*此处需要输入原密码
[oracle@ora10racn1 .ssh]$ ssh ora10racn2.ccz.com cat ~/.ssh/id_rsa.pub>>~/.ssh/authorized_keys
The authenticity of host'ora10racn2.ccz.com (192.168.0.132)' can't be established.
RSA key fingerprint isf9:5e:99:da:b1:00:9e:6b:a4:9e:a0:cd:ae:e3:4d:ca.
Are you sure you want to continue connecting(yes/no)? yes
Warning: Permanently added'ora10racn2.ccz.com,192.168.0.132' (RSA) to the list of known hosts.
[oracle@ora10racn1 .ssh]$ scp ~/.ssh/authorized_keys ora10racn2:.ssh/authorized_keys
The authenticity of host 'ora10racn2(192.168.0.132)' can't be established.
RSA key fingerprint isf9:5e:99:da:b1:00:9e:6b:a4:9e:a0:cd:ae:e3:4d:ca.
Are you sure you want to continueconnecting (yes/no)? yes
Warning: Permanently added 'ora10racn2'(RSA) to the list of known hosts.
authorized_keys 100%1221 1.2KB/s 00:00
[oracle@ora10racn1 .ssh]$ chmod 600 ~/.ssh/authorized_keys
[oracle@ora10racn1 .ssh]$ ssh ora10racn1.ccz.com date
Wed May 16 20:38:33 CST 2012
[oracle@ora10racn1 .ssh]$ ssh ora10racn2.ccz.com date
Wed May 16 20:38:38 CST 2012
[oracle@ora10racn1 .ssh]$
[oracle@ora10racn1 .ssh]$ exec /usr/bin/ssh-agent $SHELL
[oracle@ora10racn1 .ssh]$ /usr/bin/ssh-add
Identity added:/home/oracle/.ssh/id_rsa (/home/oracle/.ssh/id_rsa)
[oracle@ora10racn1 .ssh]$ ssh ora10racn1.ccz.com "date;hostname"
Wed May 16 20:52:16 CST 2012
ora10racn1.ccz.com
[oracle@ora10racn1 .ssh]$ ssh ora10racn2.ccz.com "date;hostname"
Wed May 16 20:52:21 CST 2012
ora10racn2.ccz.com
Ø 在节点2上：
[root@ora10racn2scripts]# su - oracle
[oracle@ora10racn2 ~]$ mkdir -p ~/.ssh
[oracle@ora10racn2 ~]$chmod 700 ~/.ssh
[oracle@ora10racn2 ~]$/usr/bin/ssh-keygen -t rsa
Generatingpublic/private rsa key pair.
Enter file in which tosave the key (/home/oracle/.ssh/id_rsa):
Enter passphrase(empty for no passphrase):
Enter same passphraseagain:
Your identificationhas been saved in /home/oracle/.ssh/id_rsa.
Your public key hasbeen saved in /home/oracle/.ssh/id_rsa.pub.
The key fingerprintis:
73:ec:af:49:ad:f1:3d:af:96:3a:d5:1f:5c:a7:d0:eaoracle@ora10racn2.ccz.com
[oracle@ora10racn2.ssh]$ chmod 600 ~/.ssh/authorized_keys
[oracle@ora10racn2.ssh]$ ssh ora10racn1.ccz.com hostname
ora10racn1.ccz.com
[oracle@ora10racn2.ssh]$ ssh ora10racn1.ccz.com date
Wed May 16 20:37:53CST 2012
[oracle@ora10racn2.ssh]$ ssh ora10racn2.ccz.com date
The authenticity ofhost 'ora10racn2.ccz.com (192.168.0.132)' can't be established.
RSA key fingerprint isf9:5e:99:da:b1:00:9e:6b:a4:9e:a0:cd:ae:e3:4d:ca.
Are you sure you wantto continue connecting (yes/no)? yes
Warning: Permanentlyadded 'ora10racn2.ccz.com,192.168.0.132' (RSA) to the list of known hosts.
Wed May 16 20:38:00CST 2012
[oracle@ora10racn2.ssh]$ ssh ora10racn2.ccz.com hostname
ora10racn2.ccz.com
[oracle@ora10racn2.ssh]$ ssh ora10racn2.ccz.com date
Wed May 16 20:38:11CST 2012
[oracle@ora10racn2.ssh]$ exec /usr/bin/ssh-agent $SHELL
[oracle@ora10racn2.ssh]$ /usr/bin/ssh-add
Identity added:/home/oracle/.ssh/id_rsa (/home/oracle/.ssh/id_rsa)
[oracle@ora10racn2.ssh]$ ssh ora10racn1.ccz.com "date;hostname"
Wed May 16 20:52:36CST 2012
ora10racn1.ccz.com
[oracle@ora10racn2.ssh]$ ssh ora10racn2.ccz.com "date;hostname"
Wed May 16 20:52:41CST 2012
ora10racn2.ccz.com
if [ -t 0 ]; then
stty intr ^C
fi
#/usr/bin/system-config-securitylevel &
1) 安装ocfs2
[root@ora10racn2 Server]# pwd
/media/OL5.7 x86_64dvd 20110728/Server
[root@ora10racn2Server]# ls -l ocfs2*
-rw-r--r-- 1 rootroot 328649 Jul 26 2011ocfs2-2.6.18-274.el5-1.4.8-2.el5.x86_64.rpm
-rw-r--r-- 1 rootroot 333910 Jul 26 2011ocfs2-2.6.18-274.el5debug-1.4.8-2.el5.x86_64.rpm
-rw-r--r-- 1 rootroot 328371 Jul 26 2011ocfs2-2.6.18-274.el5xen-1.4.8-2.el5.x86_64.rpm
-rw-r--r-- 1 rootroot 457984 Sep 17 2010 ocfs2console-1.6.3-2.el5.x86_64.rpm
-rw-r--r-- 1 root root1825960 Sep 17 2010ocfs2-tools-1.6.3-2.el5.x86_64.rpm
-rw-r--r-- 1 rootroot 180899 Sep 17 2010 ocfs2-tools-devel-1.6.3-2.el5.x86_64.rpm
[root@ora10racn2Server]#rpm -Uvhocfs2-2.6.18-274.el5-1.4.8-2.el5.x86_64.rpm \
>ocfs2console-1.6.3-2.el5.x86_64.rpm \
>ocfs2-tools-1.6.3-2.el5.x86_64.rpm \
>ocfs2-2.6.18-274.el5xen-1.4.8-2.el5.x86_64.rpm
warning: ocfs2-2.6.18-274.el5-1.4.8-2.el5.x86_64.rpm:Header V3 DSA signature: NOKEY, key ID 1e5e0159
Preparing... ########################################### [100%]
package ocfs2-tools-1.6.3-2.el5.x86_64is already installed
package ocfs2console-1.6.3-2.el5.x86_64is already installed
[root@ora10racn2Server]#
2) diskable SELinux
[root@ora10racn2Server]# /usr/bin/system-config-securitylevel &
3) 在两个节点上配置OCFS2
[root@ora10racn1Server]#ocfs2console &
依次【Cluster】-->【Configure Nodes】-->【add】，在每个节点上都要增加两个节点信息，注意此时虽然IP选用的是心跳IP，但hostname用的是主机名称（即hostname命令的返回值）
配置后的结果写入到文件/etc/ocfs2/cluster.conf中
node:
ip_port = 7777
ip_address = 10.10.0.131
number = 0
name = ora10racn1.ccz.com
cluster = ocfs2
node:
ip_port = 7777
ip_address = 10.10.0.132
number = 1
name = ora10racn2.ccz.com
cluster = ocfs2
cluster:
node_count = 2
name = ocfs2
注意：如果在用ocfs2console增加节点时报以下错误：
o2cb_ctl: Unable toaccess cluster service while creating node
Could not add nodenode1
则可以先将/etc/ocfs2/cluster.conf改名，然后使用ocfs2console重新加：
此时可以查看o2cb服务：
[oracle@ora10racn1 ~]$/etc/init.d/o2cb status
Driver for"configfs": Loaded
Filesystem"configfs": Mounted
Stack glue driver:Loaded
Stack plugin"o2cb": Loaded
Driver for"ocfs2_dlmfs": Loaded
Filesystem"ocfs2_dlmfs": Mounted
Checking O2CB clusterocfs2: Online
Heartbeat deadthreshold = 31
Network idle timeout: 30000
Network keepalive delay: 2000
Network reconnect delay: 2000
Checking O2CBheartbeat: Active
4) 修改o2cb的配置（只需要在一个节点上执行），将heartbeat deadthreshold从缺省的31秒改为61秒：
[root@ora10racn1 ~]# /etc/init.d/o2cb offline ocfs2
Stopping O2CB clusterocfs2: OK
[root@ora10racn1 ~]# /etc/init.d/o2cb unload
Unmounting ocfs2_dlmfsfilesystem: OK
Unloading module"ocfs2_dlmfs": OK
Unloading module"ocfs2_stack_o2cb": OK
Unmounting configfsfilesystem: OK
Unloading module"configfs": OK
[root@ora10racn1 ~]# /etc/init.d/o2cb configure
Configuring the O2CBdriver.
This will configurethe on-boot properties of the O2CB driver.
The followingquestions will determine whether the driver is loaded on
boot. The current values will be shown in brackets('[]'). Hitting
<ENTER> withouttyping an answer will keep that current value. Ctrl-C
will abort.
Load O2CB driver onboot (y/n) [y]:
Cluster stack backingO2CB [o2cb]:
Cluster to start onboot (Enter "none" to clear) [ocfs2]:
Specify heartbeat deadthreshold (>=7) [31]: 61
Specify network idletimeout in ms (>=5000) [30000]:
Specify networkkeepalive delay in ms (>=1000) [2000]:
Specify networkreconnect delay in ms (>=2000) [2000]:
Writing O2CBconfiguration: OK
Loading filesystem"configfs": OK
Mounting configfsfilesystem at /sys/kernel/config: OK
Loading stack plugin"o2cb": OK
Loading filesystem"ocfs2_dlmfs": OK
Mounting ocfs2_dlmfsfilesystem at /dlm: OK
Setting cluster stack"o2cb": OK
Starting O2CB clusterocfs2: OK
检查修改结果：
[oracle@ora10racn1 ~]$/etc/init.d/o2cb status
Driver for"configfs": Loaded
Filesystem"configfs": Mounted
Stack glue driver:Loaded
Stack plugin "o2cb":Loaded
Driver for"ocfs2_dlmfs": Loaded
Filesystem"ocfs2_dlmfs": Mounted
Checking O2CB clusterocfs2: Online
Heartbeat deadthreshold = 61
Network idle timeout: 30000
Network keepalive delay: 2000
Network reconnect delay: 2000
Checking O2CB heartbeat:Active
5) 格式化OCFS2文件系统（只能在一个节点上执行）：
[root@ora10racn1 ~]# mkfs.ocfs2 -b 4k -C 32k -N 4 -L oracrsfiles/dev/iscsi/crs1/part1
mkfs.ocfs2 1.6.3
Cluster stack: classico2cb
Overwriting existingocfs2 partition.
Proceed (y/N): y
Label: oracrsfiles
Features: sparsebackup-super unwritten inline-data strict-journal-super
Block size: 4096 (12bits)
Cluster size: 32768(15 bits)
Volume size:2145943552 (65489 clusters) (523912 blocks)
Cluster groups: 3(tail covers 977 clusters, rest cover 32256 clusters)
Extent allocator size:4194304 (1 groups)
Journal size: 67108864
Node slots: 4
Creating bitmaps: done
Initializingsuperblock: done
Writing system files:done
Writing superblock:done
Writing backupsuperblock: 1 block(s)
Formatting Journals:done
Growing extentallocator: done
Formatting slot map:done
Formatting quotafiles: done
Writing lost+found:done
mkfs.ocfs2 successful
6) 加载ocfs2文件系统（需要在两个节点上运行）：
[root@ora10racn1 ~]# mount -t ocfs2 -o datavolume,nointr -L"oracrsfiles" /u02
[root@ora10racn1 ~]# mount
/dev/mapper/VolGroup00-LogVol00on / type ext3 (rw)
proc on /proc typeproc (rw)
sysfs on /sys typesysfs (rw)
devpts on /dev/ptstype devpts (rw,gid=5,mode=620)
/dev/sda1 on /boottype ext3 (rw)
tmpfs on /dev/shm typetmpfs (rw)
none on/proc/sys/fs/binfmt_misc type binfmt_misc (rw)
sunrpc on/var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)
192.168.2.110:/mnt/nfs4backup/nfs4backup/nfs4backupon /mnt/share type nfs(rw,hard,nointr,tcp,noac,nfsvers=3,timeo=600,rsize=32768,wsize=32768,addr=192.168.2.110)
configfs on/sys/kernel/config type configfs (rw)
ocfs2_dlmfs on /dlmtype ocfs2_dlmfs (rw)
/dev/sdd1 on/u02 type ocfs2 (rw,_netdev,datavolume,nointr,heartbeat=local)
7) 在两个节点上修改/etc/fstab以让系统启动时自动加载ocfs2文件系统：
[root@ora10racn1 ~]# vi /etc/fstab
/dev/VolGroup00/LogVol00/ ext3 defaults 1 1
LABEL=/boot /boot ext3 defaults 1 2
tmpfs /dev/shm tmpfs defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
sysfs /sys sysfs defaults 0 0
proc /proc proc defaults 0 0
/dev/VolGroup00/LogVol01swap swap defaults 0 0
/home/swap swap swapdefaults 0 0
192.168.2.110:/mnt/nfs4backup/nfs4backup/nfs4backup /mnt/share nfs rw,hard,nointr,tcp,noac,vers=3,timeo=600,rsize=32768,wsize=32768 0 0
LABEL=oracrsfiles /u02 ocfs2 _netdev,datavolume,nointr 0 0
8) 在两个节点上检查o2cb在各个运行级别上的运行设置：
[root@ora10racn1 ~]# chkconfig --list o2cb
o2cb 0:off 1:off 2:on 3:on 4:on 5:on 6:off
9) 在一个节点上检查并修正ocfs2文件系统的所有者属性及权限属性：
[root@ora10racn1 /]# ls -ld /u02
drwxr-xr-x 3 root root 3896 May 16 21:54 /u02
[root@ora10racn1 /]# chown oracle:oinstall /u02
[root@ora10racn1 /]# chmod 775 /u02
[root@ora10racn1 /]# ls -ld /u02
drwxr-xr-x 3 oracle oinstall 3896 May 16 21:54 /u02
10) 创建oracleclusterware 的相关目录（只需在一个节点上运行）：
[root@ora10racn1 ~]# mkdir -p /u02/oradata/racdb
[root@ora10racn1 ~]# chown -R oracle:oinstall /u02/oradata
[root@ora10racn1 ~]# chmod -R 775 /u02/oradata
[root@ora10racn1 ~]# ls -l /u02/oradata
total 0
drwxr-xr-x 2 oracleoinstall 3896 May 16 22:01 racdb
1) 安装，其中前两个包在OLE5.7的介质中有，可直接安装，第三个需要到oracle网站下载：
[root@ora10racn1Server]# rpm -Uvhoracleasm-2.6.18-274.el5-2.0.5-1.el5.x86_64.rpm \
oracleasm-support-2.1.7-1.el5.x86_64.rpm
warning:oracleasm-2.6.18-274.el5-2.0.5-1.el5.x86_64.rpm: Header V3 DSA signature:NOKEY, key ID 1e5e0159
Preparing... ########################################### [100%]
1:oracleasm-support ########################################### [ 50%]
2:oracleasm-2.6.18-274.el###########################################[100%]
[root@ora10racn1Server]# cd /home/oracle
[root@ora10racn1oracle]# ls
database Desktop oracleasmlib-2.0.4-1.el5.x86_64.rpm
[root@ora10racn1oracle]# rpm -Uvh oracleasmlib-2.0.4-1.el5.x86_64.rpm
warning:oracleasmlib-2.0.4-1.el5.x86_64.rpm: Header V3 DSA signature: NOKEY, key ID 1e5e0159
Preparing... ########################################### [100%]
1:oracleasmlib ########################################### [100%]
2) 在两个节点上配置ASMLib：
[root@ora10racn1oracle]# /etc/init.d/oracleasm configure
Configuring the OracleASM library driver.
This will configurethe on-boot properties of the Oracle ASM library
driver. The following questions will determinewhether the driver is
loaded on boot andwhat permissions it will have. Thecurrent values
will be shown in brackets('[]'). Hitting <ENTER> withouttyping an
answer will keep thatcurrent value. Ctrl-C will abort.
Default user to ownthe driver interface []: oracle
Default group to ownthe driver interface []: oinstall
Start Oracle ASMlibrary driver on boot (y/n) [n]: y
Scan for Oracle ASMdisks on boot (y/n) [y]: y
Writing Oracle ASMlibrary driver configuration: done
Initializing theOracle ASMLib driver: [ OK ]
Scanning the systemfor Oracle ASMLib disks: [ OK ]
3) 创建ASM磁盘（只需在一个节点上执行创建，在其余节点上扫描即可）：
节点一：
[root@ora10racn1oracle]# /etc/init.d/oracleasm listdisks
[root@ora10racn1oracle]# /etc/init.d/oracleasm createdisk VOL1/dev/iscsi/asm1/part1
Marking disk"VOL1" as an ASM disk: [ OK ]
[root@ora10racn1oracle]# /etc/init.d/oracleasm createdisk VOL2 /dev/iscsi/asm2/part1
Marking disk"VOL2" as an ASM disk: [ OK ]
[root@ora10racn1oracle]# /etc/init.d/oracleasm listdisks
VOL1
VOL2
节点二：
[root@ora10racn2oracle]# /etc/init.d/oracleasm listdisks
[root@ora10racn2oracle]# /etc/init.d/oracleasm scandisks
Scanning the systemfor Oracle ASMLib disks: [ OK ]
[root@ora10racn2oracle]# /etc/init.d/oracleasm listdisks
VOL1
VOL2
1) 在两个节点上安装cvuqdisk（只有RHEL需要）
[oracle@ora10racn1 ~]$su -
Password:
[root@ora10racn1 ~]# rpm -qa --queryformat "%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n"|grep cvuqdisk
[root@ora10racn1 ~]# cd /home/oracle/ clusterware/rpm
[root@ora10racn1 rpm]#ls
cvuqdisk-1.0.1-1.rpm
[root@ora10racn1 rpm]# rpm -Uvh cvuqdisk-1.0.1-1.rpm
Preparing... ########################################### [100%]
1:cvuqdisk ########################################### [100%]
[root@ora10racn1 rpm]#scp ./cvuqdisk-1.0.1-1.rpm ora10racn2.ccz.com:/home/oracle
The authenticity ofhost 'ora10racn2.ccz.com (192.168.0.132)' can't be established.
RSA key fingerprint isf9:5e:99:da:b1:00:9e:6b:a4:9e:a0:cd:ae:e3:4d:ca.
Are you sure you wantto continue connecting (yes/no)? yes
Warning: Permanentlyadded 'ora10racn2.ccz.com,192.168.0.132' (RSA) to the list of known hosts.
root@ora10racn2.ccz.com'spassword:
cvuqdisk-1.0.1-1.rpm
[root@ora10racn2oracle]# rpm -Uvh cvuqdisk-1.0.1-1.rpm
Preparing... ###########################################[100%]
1:cvuqdisk ########################################### [100%]
2) 安装前检测（在oracle帐号下执行，且只需在一个节点上运行）：
[oracle@ora10racn1cluvfy]$ cd /home/oracle/clusterware/cluvfy
[oracle@ora10racn1cluvfy]$ mkdir -p jdk14
[oracle@ora10racn1cluvfy]$ unzip jrepack.zip -d jdk14
[oracle@ora10racn1cluvfy]$ export CV_HOME=/home/oracle/clusterware/cluvfy[oracle@ora10racn1 cluvfy]$ exportCV_JDKHOME=/home/oracle/clusterware/cluvfy/jdk14[oracle@ora10racn1 cluvfy]$ ./runcluvfy.sh stage-pre crsinst -n ora10racn1,ora10racn2 -verbose
在检测中报
Check: Userequivalence for user "oracle"
Node Name Comment
------------------------------------ ------------------------
ora10racn2 passed
ora10racn1 failed
Result: Userequivalence check failed for user "oracle".
WARNING:
User equivalence isnot set for nodes:
ora10racn1
Verification willproceed with nodes:
ora10racn2
这是因为在设置用户等效性后，第一次运行需要确认，而系统将这个确认视作失败，因此应手动先在各个节点上对所有节点（包括自身）进行下测试并确认，以确保第二次测试无需确认：
[oracle@ora10racn1 cluvfy]$ sshora10racn1 date
The authenticity ofhost 'ora10racn1 (192.168.0.131)' can't be established.
RSA key fingerprint isf9:5e:99:da:b1:00:9e:6b:a4:9e:a0:cd:ae:e3:4d:ca.
Are you sure you wantto continue connecting (yes/no)? yes
Warning: Permanentlyadded 'ora10racn1' (RSA) to the list of known hosts.
Thu May 17 08:45:40CST 2012
[oracle@ora10racn1 cluvfy]$ sshora10racn2 date
Thu May 17 08:45:55CST 2012
[oracle@ora10racn1 cluvfy]$ sshora10racn1 date
Thu May 17 08:50:10CST 2012
再次进行检测，遇到报错：
ERROR:
Could not find asuitable set of interfaces for VIPs.
Result: Nodeconnectivity check failed.
根据oracle文档（338924.1），这是一个bug，可以忽略。
3) 通过CVU对系统硬件及操作系统进行检测：
[oracle@ora10racn1 cluvfy]$ ./runcluvfy.sh stage -post hwos -n ora10racn1,ora10racn2-verbose
Performing post-checksfor hardware and operating system setup
Checking nodereachability...
......
ERROR:
Could notfind a suitable set of interfaces for VIPs.
Result: Nodeconnectivity check failed.
Checking sharedstorage accessibility...
WARNING:
Unable todetermine the sharedness of /dev/sda on nodes:
ora10racn2,ora10racn1
Disk Sharing Nodes(2 in count)
------------------------------------ ------------------------
/dev/sdb ora10racn2ora10racn1
Disk Sharing Nodes(2 in count)
------------------------------------ ------------------------
/dev/sdc ora10racn2ora10racn1
Disk Sharing Nodes(2 in count)
------------------------------------ ------------------------
/dev/sdd ora10racn2ora10racn1
Shared storage checkwas successful on nodes "ora10racn2,ora10racn1".
Post-check forhardware and operating system setup was unsuccessful on all the nodes.
检测结果中除了VIP错误（忽略），还有一个warning，这个warning是因为CVU调用了linux的smartctl，而smartctl不能返回iscsi设备的序列号所致，不影响RAC的安装和使用，忽略。
注意此处需要将crshome的路径做下修改，不要和oracle的home重合了：
根据实际情况增加节点：
根据情况调整网卡设置，其中eth1是连接存储的网卡，此处我单独设置成DoNotUse：
这只ocr及voting disk的存储路径：
注意：在执行root.sh前先如下修复下srvctl及vipca，否则报错，在实行root.sh后需要在图形界面中以root身份调用vipca配置vip，否则OUI仍然会报错：
分别在两个节点按顺序以root身份run相应脚本，但在最后一个节点run最后一个脚本root.sh时有一个报错：
CSS is active on allnodes.
Waiting for the OracleCRSD and EVMD to start
Oracle CRS stackinstalled and running under init(1M)
Running vipca(silent)for configuring nodeapps
/u01/app/crs/jdk/jre//bin/java:error while loading shared libraries: libpthread.so.0: cannot open sharedobject file: No such file or directory
经查，这是一个oracle的bug，解决方法见：
http://hi.baidu.com/heroofhero/blog/item/76747032361fc84dac4b5f09.html
修复：
Ø /u01/app/crs/bin/vipca:
152 exportLD_LIBRARY_PATH
153 ;;
154 esac
155
156 unset LD_ASSUME_KERNEL
157
158ARGUMENTS=""
159 NUMBER_OF_ARGUMENTS=$#
160 if [ $NUMBER_OF_ARGUMENTS -gt 0 ]; then
161 ARGUMENTS=$*
162 fi
/u01/app/crs/bin/srvctl:
166 #Remove this workaround when the bug 3937317 is fixed
167 LD_ASSUME_KERNEL=2.4.19
168 export LD_ASSUME_KERNEL
169 unset LD_ASSUME_KERNEL
170
171 # Run ops control utility
解决后回到原OUI中点击OK继续。
同安装clusterware后对$CRS_HOME/bin/srvctl做的修正一样，需对$ORACLE_HOME/bin/srvctl进行修正：
LD_ASSUME_KERNEL=2.4.19
exportLD_ASSUME_KERNEL
unsetLD_ASSUME_KERNEL
1) 创建前的预检：
[oracle@ora10racn1cluvfy]$ env|grep CV
CV_JDKHOME=/home/oracle/clusterware/cluvfy/jdk14
CV_HOME=/home/oracle/clusterware/cluvfy
[oracle@ora10racn1cluvfy]$ pwd
/home/oracle/clusterware/cluvfy
[oracle@ora10racn1 cluvfy]$ ./runcluvfy.sh stage-pre dbcfg -n ora10racn1,ora10racn2 -d ${ORACLE_HOME} -verbose
预检中除了前面出现过的VIP问题以外，没有其他问题，该VIP问题可以忽略。
2) 用dbca创建数据库：
在创建的过程中注意，ASM的spfile缺省位置由于不是共享存储，应改成共享存储位置，如图：
在创建ASM实例时报错：ORA-27125:unable to create shared memory segment
解决的办法是将dba的用户组加入到文件/proc/sys/vm/hugetlb_shm_group
[root@rac2 ~]# idoracle
uid=500(oracle)gid=501(oinstall)groups=501(oinstall),502(dba),503(asmadmin),504(oper)
[root@rac2 ~]#more/proc/sys/vm/hugetlb_shm_group
0
下面用root执行下面的命令，将dba组添加到系统内核中：
[root@rac2 ~]# echo502 >/proc/sys/vm/hugetlb_shm_group
具体参见：http://blog.csdn.net/tianlesoftware/article/details/7309046
3) 创建磁盘组时注意修改下磁盘路径:
如使用缺省的ORCL:VOL*，将会导致机器HANG住。
4) 创建成功：
[oracle@ora10racn1 ~]$dbca&
注意新建的taf服务不要带域名，否则会报错。
SQL> show parameterservice
NAME TYPE VALUE
----------------------------------------------- ------------------------------
service_names string racdb.ccz.com, racdb_taf
[oracle@ora10racn2disks]$ crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora....SM1.asmapplication ONLINE ONLINE ora10racn1
ora....N1.lsnrapplication ONLINE ONLINE ora10racn1
ora....cn1.gsdapplication ONLINE ONLINE ora10racn1
ora....cn1.onsapplication ONLINE ONLINE ora10racn1
ora....cn1.vipapplication ONLINE ONLINE ora10racn1
ora....SM2.asmapplication ONLINE ONLINE ora10racn2
ora....N2.lsnrapplication ONLINE ONLINE ora10racn2
ora....cn2.gsdapplication ONLINE ONLINE ora10racn2
ora....cn2.onsapplication ONLINE ONLINE ora10racn2
ora....cn2.vipapplication ONLINE ONLINE ora10racn2
ora.racdb.db application ONLINE ONLINE ora10racn1
ora....b1.instapplication ONLINE ONLINE ora10racn1
ora....b2.instapplication ONLINE ONLINE ora10racn2
ora...._taf.csapplication ONLINE ONLINE ora10racn1
ora....db1.srvapplication ONLINE ONLINE ora10racn1
ora....db2.srvapplication ONLINE ONLINE ora10racn2
设置taf服务在系统启动时自动启动：
[oracle@ora10racn2bash]$ srvctl enable service -d racdb -s racdb_taf
查看taf服务的状态：
[oracle@ora10racn2bash]$srvctl config service -d racdb -s racdb_taf -a
racdb_tafPREF: racdb1 racdb2 AVAIL: TAF: basic
$ sqlplus / as sysdba
SQL>@?/rdbms/admin/utlrp.sql
1) 在一号节点上关闭RAC设置：
SYS@racdb1 SQL>showparameter cluster_database
NAME TYPE VALUE
----------------------------------------------- ------------------------------
cluster_database boolean TRUE
cluster_database_instances integer 2
SYS@racdb1SQL>alter system set cluster_database=falsescope=spfile sid='racdb1';
2) shutdown RAC上所有实例：
[oracle@ora10racn1admin]$ srvctl stop database -d racdb
[oracle@ora10racn1admin]$ crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora....SM1.asmapplication ONLINE ONLINE ora10racn1
ora....N1.lsnr application ONLINE ONLINE ora10racn1
ora....cn1.gsdapplication ONLINE ONLINE ora10racn1
ora....cn1.onsapplication ONLINE ONLINE ora10racn1
ora....cn1.vipapplication ONLINE ONLINE ora10racn1
ora....SM2.asm application ONLINE ONLINE ora10racn2
ora....N2.lsnrapplication ONLINE ONLINE ora10racn2
ora....cn2.gsdapplication ONLINE ONLINE ora10racn2
ora....cn2.onsapplication ONLINE ONLINE ora10racn2
ora....cn2.vipapplication ONLINE ONLINE ora10racn2
ora.racdb.db application OFFLINE OFFLINE
ora....b1.instapplication OFFLINE OFFLINE
ora....b2.instapplication OFFLINE OFFLINE
ora...._taf.csapplication OFFLINE OFFLINE
ora....db1.srvapplication ONLINE OFFLINE
ora....db2.srvapplication ONLINE OFFLINE
3) 在1号节点上启动实例到mount状态：
SYS@racdb1 SQL>startup mount;
ORACLE instancestarted.
Total System GlobalArea 1862270976 bytes
Fixed Size 2021600 bytes
Variable Size 469763872 bytes
Database Buffers 1375731712 bytes
Redo Buffers 14753792 bytes
Database mounted.
SYS@racdb1SQL>archive log list;
Database log mode No Archive Mode
Automaticarchival Disabled
Archivedestination USE_DB_RECOVERY_FILE_DEST
Oldest online logsequence 33
Current logsequence 34
4) 在1号节点上打开archive模式：
SYS@racdb1SQL>alter database archivelog;
Database altered.
5) 在1号节点上enable cluster的设置并关闭实例：
SYS@racdb1SQL>alter system set cluster_database=true scope=spfile sid='racdb1';
System altered.
SYS@racdb1SQL>shutdown immediate;
ORA-01109: databasenot open
Database dismounted.
ORACLE instance shutdown.
6) 启动所有实例并验证：
[oracle@ora10racn1admin]$ srvctl start database -d racdb
[oracle@ora10racn1admin]$ sqlplus /nolog
SQL*Plus: Release10.2.0.1.0 - Production on Fri May 18 17:44:53 2012
Copyright (c) 1982,2005, Oracle. All rights reserved.
@ SQL>conn / assysdba
Connected.
SYS@racdb1SQL>archive log list;
Database log mode ArchiveMode
Automaticarchival Enabled
Archivedestination USE_DB_RECOVERY_FILE_DEST
Oldest online logsequence 33
Next log sequence toarchive 34
Current logsequence 34
1) 将一个节点上的ASM及RDBMS的口令文件移到共享文件系统上：
oracle@ora10racn2dbs]$ cd $ORACLE_HOME/dbs
[oracle@ora10racn2dbs]$ ls
ab_+ASM2.dat hc_+ASM2.dat hc_racdb2.dat init+ASM2.ora initdw.ora init.ora initracdb2.ora orapw+ASM2 orapwracdb2
[oracle@ora10racn2dbs]$ mv ./orapw+ASM2 /u02/oradata/racdb/dbs/orapw+ASM
[oracle@ora10racn2dbs]$ mv ./orapwracdb2/u02/oradata/racdb/dbs/orapwracdb
2) 在各个节点建立到共享口令文件的软连接：
[oracle@ora10racn2dbs]$ pwd
/u01/app/oracle/product/10.2.0/db_1/dbs
[oracle@ora10racn2dbs]$ ln -s /u02/oradata/racdb/dbs/orapw+ASM/u01/app/oracle/product/10.2.0/db_1/dbs/orapw+ASM2
[oracle@ora10racn2dbs]$ ln -s /u02/oradata/racdb/dbs/orapwracdb/u01/app/oracle/product/10.2.0/db_1/dbs/orapwracdb2
[oracle@ora10racn2dbs]$ ls -l
total 48
-rw-rw---- 1 oracleoinstall 1571 May 18 15:00 ab_+ASM2.dat
-rw-rw---- 1 oracleoinstall 1552 May 18 15:00 hc_+ASM2.dat
-rw-rw---- 1 oracleoinstall 1552 May 18 17:44 hc_racdb2.dat
-rw-r----- 1 oracleoinstall 47 May 18 14:59 init+ASM2.ora
-rw-r----- 1 oracleoinstall 12920 May 3 2001 initdw.ora
-rw-r----- 1 oracleoinstall 8385 Sep 11 1998 init.ora
-rw-r----- 1 oracleoinstall 39 May 18 15:33initracdb2.ora
lrwxrwxrwx 1 oracleoinstall 32 May 19 08:22 orapw+ASM2-> /u02/oradata/racdb/dbs/orapw+ASM
lrwxrwxrwx 1 oracleoinstall 33 May 19 08:22 orapwracdb2-> /u02/oradata/racdb/dbs/orapwracdb
[oracle@ora10racn1dbs]$ rm -r orapw+ASM1
[oracle@ora10racn1dbs]$ rm -r orapwracdb1
[oracle@ora10racn1dbs]$ ln -s /u02/oradata/racdb/dbs/orapw+ASM/u01/app/oracle/product/10.2.0/db_1/dbs/orapw+ASM1
[oracle@ora10racn1dbs]$ ln -s /u02/oradata/racdb/dbs/orapwracdb/u01/app/oracle/product/10.2.0/db_1/dbs/orapwracdb1
[oracle@ora10racn1dbs]$ ls -l
total 48
-rw-rw---- 1 oracleoinstall 796 May 18 14:59 ab_+ASM1.dat
-rw-rw---- 1 oracleoinstall 1552 May 18 14:59 hc_+ASM1.dat
-rw-rw---- 1 oracleoinstall 1552 May 18 17:43 hc_racdb1.dat
-rw-r----- 1 oracleoinstall 47 May 18 14:59 init+ASM1.ora
-rw-r----- 1 oracleoinstall 12920 May 3 2001 initdw.ora
-rw-r----- 1 oracleoinstall 8385 Sep 11 1998 init.ora
-rw-r----- 1 oracleoinstall 39 May 18 15:33initracdb1.ora
lrwxrwxrwx 1 oracleoinstall 32 May 19 08:27 orapw+ASM1-> /u02/oradata/racdb/dbs/orapw+ASM
lrwxrwxrwx 1 oracleoinstall 33 May 19 08:27 orapwracdb1-> /u02/oradata/racdb/dbs/orapwracdb
NAME BYTES/(1024*1024)
-------------------------------------------------------------------
+ORADAT/racdb/datafile/system.259.783615899 480
+ORADAT/racdb/datafile/undotbs1.260.783615911 330
+ORADAT/racdb/datafile/sysaux.261.783615915 270
+ORADAT/racdb/datafile/undotbs2.263.783615923 200
+ORADAT/racdb/datafile/users.264.783615929 5
SYS@racdb2SQL>alter database datafile '+ORADAT/racdb/datafile/users.264.783615929'resize 1024M;
Database altered.
SYS@racdb2SQL>alter tablespace users add datafile '+ORADAT' size 1024m autoextend on;
Tablespace altered.
SYS@racdb2SQL>create tablespace indx datafile '+ORADAT' size 1024M
2 autoextend on next 50M maxsize unlimited
3* extent management local autoallocatesegment space management auto；
SYS@racdb2SQL>alter database datafile '+ORADAT/racdb/datafile/system.259.783615899'resize 800M;
Database altered.
SYS@racdb2SQL>alter database datafile '+ORADAT/racdb/datafile/sysaux.261.783615915'resize 500M;
Database altered.
SYS@racdb2SQL>alter database datafile '+ORADAT/racdb/datafile/undotbs1.260.783615911'resize 1024m;
Database altered.
SYS@racdb2SQL>alter database datafile '+ORADAT/racdb/datafile/undotbs2.263.783615923'resize 1024M;
Database altered.
SYS@racdb2SQL>select name,bytes/(1024*1024) from v$datafile;
NAME BYTES/(1024*1024)
-------------------------------------------------------------------
+ORADAT/racdb/datafile/system.259.783615899 800
+ORADAT/racdb/datafile/undotbs1.260.783615911 1024
+ORADAT/racdb/datafile/sysaux.261.783615915 500
+ORADAT/racdb/datafile/undotbs2.263.783615923 1024
+ORADAT/racdb/datafile/users.264.783615929 1024
+ORADAT/racdb/datafile/users.268.783679533 1024
+ORADAT/racdb/datafile/indx.269.783679683 1024
SYS@racdb2SQL>select name,bytes/(1024*1024) from v$tempfile;
NAME BYTES/(1024*1024)
-------------------------------------------------------------------
+ORADAT/racdb/tempfile/temp.262.783615919 28
SYS@racdb2SQL>alter database tempfile '+ORADAT/racdb/tempfile/temp.262.783615919'resize 1024M;
Database altered.