###############################################################
### ###
### Create Package with RM (CA) installed - metro cluster: ###
### ###
###############################################################
grep -v CVS | grep dsk | awk '{print $1}' > all_rdsk.txt
$ sed 's/rdsk/dsk/' all_rdsk.txt > all_dsk.txt
$ strings /etc/lvmtab | grep dsk > used_dsk.txt
# Remove entries for VG00 from used_dsk.txt
### with $ strings /etc/lvmtab # you can check wich disks are in vg00
$ grep -vf used_dsk.txt all_dsk.txt > new_dsk.txt
### Next we check if the disk in new_dsk.txt file are alocated by
### any volume group. If the answer is like this:
### DISK /dev/dsk/c12t1d4
### 0000000 0 0 0 0
### 0000010
### The disks are unused (free for us).
$ for i in `cat new_dsk.txt`
do
echo DISK $i
xd -j8200 -N16 -tu $i
done
grep -v CVS | grep dsk | awk '{print $1}' > all_rdsk.txt
$ sed 's/rdsk/dsk/' all_rdsk.txt > all_dsk.txt
$ strings /etc/lvmtab | grep dsk > used_dsk.txt
# Remove entries for VG00 from used_dsk.txt
### with $ strings /etc/lvmtab # you can check wich disks are in vg00
$ grep -vf used_dsk.txt all_dsk.txt > new_dsk.txt
### 3. (Optional) Create users for oracle on both nodes:
### Find unused ID's for user/group on both nodes. For example 111
### User: orasas1
### Group: dbasas1
$ groupadd -g 111 dbasas1
$ useradd -m -g dbasas1 -d /opt/app/orasas1 -u 111 orasas1
$ passwd orasas1
grep 06:
# /dev/rdsk/c14t1d4 cd 01 0c CL2A 06:00 OPEN-E 00060083
### /dev/rdsk/c22t1d4 ef 01 0c CL1A 06:00 OPEN-E 00060083
### /dev/rdsk/c12t1d4 98 01 0c CL2E 06:01 OPEN-E 00060083
### /dev/rdsk/c20t1d4 b2 01 0c CL1E 06:01 OPEN-E 00060083
### /dev/rdsk/c18t1d4 55 01 0c CL2J 06:02 OPEN-E 00060083
### /dev/rdsk/c26t1d4 72 01 0c CL1J 06:02 OPEN-E 00060083
### /dev/rdsk/c16t1d4 cd 01 0c CL2N 06:03 OPEN-E 00060083
### /dev/rdsk/c24t1d4 ef 01 0c CL1N 06:03 OPEN-E 00060083
### /dev/rdsk/c22t1d5 ef 01 0d CL1A 06:04 OPEN-E 00060083
### /dev/rdsk/c14t1d5 cd 01 0d CL2A 06:04 OPEN-E 00060083
### /dev/rdsk/c20t1d5 b2 01 0d CL1E 06:05 OPEN-E 00060083
### /dev/rdsk/c12t1d5 98 01 0d CL2E 06:05 OPEN-E 00060083
### /dev/rdsk/c18t1d5 55 01 0d CL2J 06:06 OPEN-E 00060083
### /dev/rdsk/c26t1d5 72 01 0d CL1J 06:06 OPEN-E 00060083
### /dev/rdsk/c24t1d5 ef 01 0d CL1N 06:07 OPEN-E 00060083
### /dev/rdsk/c16t1d5 cd 01 0d CL2N 06:07 OPEN-E 00060083
###
### So the command in my case will looks like:
$ for i in /dev/rdsk/c14t1d4 /dev/rdsk/c12t1d4 /dev/rdsk/c18t1d4 /dev/rdsk/c16t1d4 /dev/rdsk/c22t1d5 /dev/rdsk/c20t1d5 /dev/rdsk/c18t1d5 /dev/rdsk/c24t1d5
do
pvcreate $i
done
### Make volume group directroy:
$ mkdir /dev/vg_SAS1
$ mknod /dev/vg_SAS1/group c 64 0x070000
### Then we create volume group itself. Notice that in order to
### balance loading of FCs we will create volumte group with pvn,
### and the command will look like:
### $ vgcreate - l 128 -p 128 -s 16 -g <pvg_name> /dev/dsk/<First Fisk First Channe> /dev/dsk/<Second Disc Second Channel> …
$ vgcreate -l 128 -p 128 -s 16 -g pvg_SAS1 vg_SAS1 /dev/dsk/c14t1d4 /dev/dsk/c20t1d4 /dev/dsk/c18t1d4 /dev/dsk/c24t1d4 /dev/dsk/c22t1d4 /dev/dsk/c12t1d4 /dev/dsk/c26t1d4 /dev/dsk/c16t1d4
### 6. Create Logical Volumes in Volume group
### lvcreate -D y -s g -r n -L 10240 vg_SAS1
### -D - distributed (y = yes).
### -s Set strict allocation policy:
### g -> Mirrors of a logical extend cannot share
### the same PV group. A PVG allocation policy
### cannot be set on a logical volume in a
### volume group that does not have a phisical
### volume group (PVG) defined.
### -r Set the logical volume bad block relocation
### policy. This is an obsolet flag available only
### to provide compatibility with prior HP-UX release.
### -L Allocate space to the logical volume in megabytes.
$ lvcreate -D y -s g -r n -L 3072 -n lvol1 vg_SAS1
$ lvcreate -D y -s g -r n -L 1024 -n lvol2 vg_SAS1
$ lvcreate -D y -s g -r n -L 1024 -n lvol3 vg_SAS1
$ lvcreate -D y -s g -r n -L 10240 -n lvol4 vg_SAS1
$ lvcreate -D y -s g -r n -L 1024 -n lvol5 vg_SAS1
### 7. Create filesystem for new create Logical Volumes
$ newfs -F vxfs -o largefiles /dev/vg_SAS1/rlvol1
$ newfs -F vxfs -o largefiles /dev/vg_SAS1/rlvol2
$ newfs -F vxfs -o largefiles /dev/vg_SAS1/rlvol3
$ newfs -F vxfs -o largefiles /dev/vg_SAS1/rlvol4
$ newfs -F vxfs -o largefiles /dev/vg_SAS1/rlvol5
### 8. Creating mountpoings for new file systems
$ mkdir -p /opt/app/orasas1
$ mkdir -p /var/opt/orasas1/oraoriglog/TRUTH/
$ mkdir -p /var/opt/orasas1/oraarch1/TRUTH/
$ mkdir -p /var/opt/orasas1/oradata1/TRUTH/
$ mkdir -p /var/opt/orasas1/oratemp/TRUTH/
### Changing premitions for mountpoings:
$ chown -R orasas1:dbasas1 /opt/app/orasas1
$ chown -R orasas1:dbasas1 /var/opt/orasas1/
### 9. Configuring Volume Group to be cluster aware:
$ vgchange -a n /dev/vg_SAS1
$ vgchange -c y /dev/vg_SAS1
$ vgchange -a e /dev/vg_SAS1
$ vgcfgbackup /dev/vg_SAS1
$ vgexport -s -p -m vg_SAS1.map /dev/vg_SAS1
### 10. Choose free HORCM instance number
### Perform this check on both nodes
$ grep INST /etc/cmcluster/*/*xp*.env
### I choose HORCMINST = 5
### Here new_rdsk.txt file contains only the disks which are included
### in volume group:
### /dev/rdsk/c14t1d4
### /dev/rdsk/c22t1d4
### /dev/rdsk/c12t1d4
### /dev/rdsk/c20t1d4
### /dev/rdsk/c18t1d4
### /dev/rdsk/c26t1d4
### /dev/rdsk/c16t1d4
### /dev/rdsk/c24t1d4
###
$ cat new_rdsk.txt | /HORCM/usr/bin/mkconf.sh -g vg_SAS1 -i 5
horcm5.conf|
# +-------------+
# Created by mkconf.sh on Thu Feb 14 10:37:35 MET 2008 |
# | |
HORCM_MON |
# | #ip_address service poll(10ms) timeout(10ms) |
NONE horcm5 1000 3000 |
# | #^edit_here ^edit_here |
… |
# | … |
HORCM_INST |
# | #dev_group ip_address service |
vg_BAC iwwrim.gre.hp.com horcm5 |
# | # ^IP of the other node |
### +-------------+
###
$ vi horcm5.conf
$ cp horcm5.conf /HORCM/etc/
$ ln -s /HORCM/etc/horcm5.conf /etc/horcm5.conf
### Add entry to /etc/services for this service. Use unique port (2410 in this example)
$ vi /etc/services
### Add line like this:
### horcm5 2410/udp #RaidManager instance 5
### Start HORCM:
$ horcmstart.sh 5
$ export HORCMINST=5
### 11. On the secondary node:
### Add entry to /etc/services, as one in first node:
$ vi /etc/services
### Add line like this:
### horcm5 2410/udp #RaidManager instance 5
$ sed -e 's/dsk/rdsk/' new_dsk.txt > new_rdsk.txt
### Edit new_rdsk.txt to include only this entries, which are included
### into Volume Group:
### /dev/rdsk/c44t1d4
### /dev/rdsk/c34t1d4
### /dev/rdsk/c47t1d4
### /dev/rdsk/c37t1d4
### /dev/rdsk/c50t1d4
### /dev/rdsk/c40t1d4
### /dev/rdsk/c42t1d4
### /dev/rdsk/c52t1d4
###
$ cat new_rdsk.txt | /HORCM/usr/bin/mkconf.sh -g vg_SAS1 -i 5
### Please check '/root/home/hpeyoyov/package_create/horcm5.conf',
### '/root/home/hpeyoyov/package_create/log5/curlog/horcm_*.log',
### and modify 'ip_address & service'.
###
horcm5.conf|
# +-------------+
# Created by mkconf.sh on Thu Feb 14 11:42:14 MET 2008 |
# | |
HORCM_MON |
# | #ip_address service poll(10ms) timeout(10ms) |
NONE horcm5 1000 3000 |
# | … |
… |
# | HORCM_INST |
#dev_group ip_address service |
# | vg_BAC iwwrim.gre.hp.com horcm5 |
# ^IP of the other node |
# +-------------+
###
$ vi horcm5.conf
$ cp horcm5.conf /HORCM/etc/
$ ln -s /HORCM/etc/horcm5.conf /etc/horcm5.conf
### Start HORCM service for instance 5
$ horcmstart.sh 5
### 12. Go back to NODE1:
### Check pair status
$ pairdisplay -g vg_SAS1
### Create pairs:
$ paircreate -g vg_SAS1 -f async -vl -c 15
### Check with pairdisplay until status is like below:
$ pairdisplay -g vg_SAS1 -fc -CLI
### Group PairVol(L/R) (Port#,TID, LU),Seq#,LDEV#.P/S,Status,Fence,Seq#,P-LDEV# M
### vg_SAS1 vg_SAS1_000(L) (CL2-A , 8, 1)60083 1536.P-VOL PAIR ASYNC ,60080 3072 -
### vg_SAS1 vg_SAS1_000(R) (CL3-A ,16, 0)60080 3072.S-VOL PAIR ASYNC ,- 1536 -
### vg_SAS1 vg_SAS1_001(L) (CL2-E , 4, 0)60083 1537.P-VOL PAIR ASYNC ,60080 3073 -
### vg_SAS1 vg_SAS1_002(L) (CL2-J , 2, 1)60083 1538.P-VOL PAIR ASYNC ,60080 3074 -
### vg_SAS1 vg_SAS1_002(R) (CL3-J ,14, 2)60080 3074.S-VOL PAIR ASYNC ,- 1538 -
### vg_SAS1 vg_SAS1_003(L) (CL2-N ,12, 7)60083 1539.P-VOL PAIR ASYNC ,60080 3075 -
### vg_SAS1 vg_SAS1_003(R) (CL4-N ,13, 7)60080 3075.S-VOL PAIR ASYNC ,- 1539 -
### 13. On NODE2:
$ mkdir /dev/vg_SAS1
$ mknod /dev/vg_SAS1/group c 64 0x070000
$vgimport -v -m vg_SAS1.map /dev/vg_SAS1 /dev/dsk/c44t1d4 /dev/dsk/c37t1d4 /dev/dsk/c50t1d4 /dev/dsk/c52t1d4 /dev/dsk/c34t1d4 /dev/dsk/c47t1d4 /dev/dsk/c40t1d4 /dev/dsk/c42t1d4
/etc/lvmpvg |
# +-------------+
VG /dev/vg_SAS1 |
# | PVG pvg_SAS1 |
/dev/dsk/c44t1d4 |
# | /dev/dsk/c37t1d4 |
/dev/dsk/c50t1d4 |
# | /dev/dsk/c52t1d4 |
/dev/dsk/c34t1d4 |
# | /dev/dsk/c47t1d4 |
/dev/dsk/c40t1d4 |
# | /dev/dsk/c42t1d4 |
### +-------------+
###
### Activate Volume Group
$ vgchange -a r vg_SAS1
### -a r - If a volume group is marked as part
### of a high availability cluster, the
### high availability software must be
### running on the system; otherwise,
### the volume group is not activated.
$ vgcfgbackup /dev/vg_SAS1
$ vgchange -a n /dev/vg_SAS1
### Create mountpoints:
$ mkdir -p /opt/app/orasas1
$ mkdir -p /var/opt/orasas1/oraoriglog/TRUTH/
$ mkdir -p /var/opt/orasas1/oraarch1/TRUTH/
$ mkdir -p /var/opt/orasas1/oradata1/TRUTH/
$ mkdir -p /var/opt/orasas1/oratemp/TRUTH/
### ??? To check if I need to change ownership on above created direcotries!
### 14. Package creation:
### On the primary node:
$ mkdir /etc/cmcluster/sasdb1
$ cd /etc/cmcluster/sasdb1
$ cmmakepkg -p sasdb1.conf
$ cmmakepkg -s sasdb1.sh
sasdb1.conf |
# +-------------+
PACKAGE_NAME sasdb1 |
# | |
PACKAGE_TYPE FAILOVER |
# | |
FAILOVER_POLICY CONFIGURED_NODE |
# | |
FAILBACK_POLICY MANUAL |
# | |
NODE_NAME gwwrim |
# | NODE_NAME iwwrim |
|
# | AUTO_RUN YES |
|
# | LOCAL_LAN_FAILOVER_ALLOWED YES |
|
# | NODE_FAIL_FAST_ENABLED NO | |
RUN_SCRIPT /etc/cmcluster/sasdb1/sasdb1.sh |
# | RUN_SCRIPT_TIMEOUT NO_TIMEOUT |
HALT_SCRIPT /etc/cmcluster/sasdb1/sasdb1.sh |
# | HALT_SCRIPT_TIMEOUT NO_TIMEOUT |
|
# | SUBNET 15.143.138.0 |
|
# +-------------+
sasdb1.sh |
# +-------------+
… |
# | VG[0]="vg_SAS1" |
… |
# | IP[0]="15.143.138.196" |
SUBNET[0]="15.143.138.0" # netstat -in to check netmask |
# | |
IP[1]="16.44.41.49" |
# | SUBNET[1]="16.44.40.0" # netstat -in to check netmask |
… |
# | LV[0]="/dev/vg_SAS1/lvol1"; FS[0]="/opt/app/orasas1"; FS_MOUNT_OPT[0]="-o rw,delaylog,largefiles"; FS_UMOUNT_OPT[0]=""; FS_FSCK_OPT[0]=""; FS_TYPE[0]="vxfs";
LV[1]="/dev/vg_SAS1/lvol2"; FS[1]="/var/opt/orasas1/oraoriglog/TRUTH"; FS_MOUNT_OPT[1]="-o rw,delaylog,largefiles"; FS_UMOUNT_OPT[1]=""; FS_FSCK_OPT[1]=""; FS_TYPE[1]="vxfs";
# | LV[2]="/dev/vg_SAS1/lvol3"; FS[2]="/var/opt/orasas1/oraarch1/TRUTH"; FS_MOUNT_OPT[2]="-o rw,delaylog,largefiles"; FS_UMOUNT_OPT[2]=""; FS_FSCK_OPT[2]=""; FS_TYPE[2]="vxfs";
LV[3]="/dev/vg_SAS1/lvol4"; FS[3]="/var/opt/orasas1/oradata1/TRUTH"; FS_MOUNT_OPT[3]="-o rw,delaylog,largefiles"; FS_UMOUNT_OPT[3]=""; FS_FSCK_OPT[3]=""; FS_TYPE[3]="vxfs";
# | LV[4]="/dev/vg_SAS1/lvol5"; FS[4]="/var/opt/orasas1/oratemp/TRUTH"; FS_MOUNT_OPT[4]="-o rw,delaylog,largefiles"; FS_UMOUNT_OPT[4]=""; FS_FSCK_OPT[4]=""; FS_TYPE[4]="vxfs";
### + -------------——+
### Check if package IP address are resolvin correctly:
### You should recieve thess addresses from erquestor
$ nslookup 15.143.138.196
$ nslookup 16.44.41.49
### Add entries in /etc/hosts
### 15.143.138.196 sasdb1.gre.hp.com sasdb1
### 16.44.41.49 sasdb1-b.gre.hp.com sasdb1-b
sasdb1_xpca.env |
# +-------------+
#!/usr/bin/sh |
# | #"@(#) A.04.22 MCCA - Last modified 10/29/02 $" |
#Copyright (c) Hewlett-Packard Company, 1999, 2000, 2001, 2002, |
# | #2003, 2004 |
|
# | AUTO_PSUEPSUS=0 |
AUTO_FENCEDATA_SPLIT=1 |
# | AUTO_SVOLPSUS=0 |
AUTO_SVOLPSUE=0 |
# | AUTO_SVOLPFUS=0 |
AUTO_PSUSSSWS=0 |
# | AUTO_NONCURDATA=0 |
MULTIPLE_PVOL_OR_SVOL_FRAMES_FOR_PKG=0 |
# | HORCMPERM=MGRNOINST |
HORCMINST=5 # change here |
# | HORCTIMEOUT=360 |
WAITTIME=300 |
# | PKGDIR="/etc/cmcluster/sasdb1" # change here |
FENCE="ASYNC" # change here |
# | DEVICE_GROUP="vg_SAS1" # change here |
CLUSTER_TYPE="metro" |
# +-------------+
### 15. On NODE2 secondary:
$ mkdir /etc/cmcluster/sasdb1/
### 16. On NODE1 (primary):
$ scp /etc/cmcluster/sasdb1/* iwwrim:/etc/cmcluster/sasdb1/ # iwwrim is NODE2 (secondary)
### You can check that cmclconfig.ascii file is recent with "cmgetconf" command and
### compare output of the command with existing file
$ cmcheckconf -v -C ../cmclconfig.ascii -P sasdb1.conf
###
$ cmviewcl -v
$ cmapplyconf -k -v -C ../cmclconfig.ascii -P sasdb1.conf
$ cmviewcl -v
### 17. Check the configuration
### to run cluster node use the next command:
$ cmrunpkg -v sasdb1
