Rocks 6.1 – IPoIB

#
# Commands below need to be run from the front end as root.
# Front end usually has the IP address 10.1.1.1 for eth0.
 
#
# Network
rocks add network ibnet subnet=10.2.0.0 netmask=255.255.0.0
rocks set host interface ip     localhost iface=ib0 ip=10.2.1.1
rocks set host interface subnet localhost iface=ib0 subnet=ibnet
rocks set host interface module localhost iface=ib0 module=ip_ipoib
rocks set host interface name   localhost iface=ib0 name=wigner
rocks sync host network localhost
 
#
# Firewall
rocks add firewall host=localhost chain=INPUT protocol=all service=all action=ACCEPT network=ibnet iface=ib0 rulename="A80-IB0-PRIVATE"
rocks sync host firewall localhost
 
#
# /etc/hosts.local
echo "# Front end" > /etc/hosts.local
printf "%-14s  %-20s  %-14s\n"  "10.2.1.1"  "wigner.ibnet" "ib-wigner" >> /etc/hosts.local

#! /bin/bash
#
# BASH script to configure IPoIB for non front end nodes in a Rocks 6.1 cluster.
# IP address scheme for InfiniBand (ib0) will reflect that of Ethernet (eth0).
# Must be root to run this script.
 
# Function to convert the first character in a string to uppercase
function ucfirst_character () {
  original_string="$@"               
  first_character=${original_string:0:1}   
  rest_of_the_string=${original_string:1}       
  first_character_uc=`echo "$first_character" | tr a-z A-Z`
  echo "${first_character_uc}${rest_of_the_string}"  
}
 
 
# Necessary variables 
# Remove login and/or nas from the list below if the cluster does not have login and/or NAS nodes
export MYNODETYPES="login nas compute"
 
# Outer for loop begins
for x in $MYNODETYPES
do
 
  # List of nodes of given type (login, nas or compute)
  export MYNODES=`rocks list host | grep "$x" | awk -F ':' '{ print $1 }' | sort -t- -k 2,2n -k 3,3n`
 
  # /etc/hosts.local header for a given type of node
  if [ $x == "nas" ]
  then
    export y=`echo $x | tr a-z A-Z`
  else
    export y=`ucfirst_character $x`
  fi
  echo "# $y node(s)" >> /etc/hosts.local
 
  # Inner for loop begins
  for MYHOSTNAME_ETH0 in $MYNODES
  do
    #
    # Additiional necessary variables
    export MYHOSTNAME_IB0="ib-$MYHOSTNAME_ETH0"
    export MYHOSTIP_ETH0=`rocks list host interface $MYHOSTNAME_ETH0 | grep "eth0" | awk '{ print $4 }'`
    export MYHOSTIP_IB0=`echo $MYHOSTIP_ETH0 | sed 's/10.1/10.2/g'`
    export MYSHORTNAME_ETH0=`echo $MYHOSTNAME_ETH0 | sed 's/compute/c/g' | sed 's/login/l/g' | sed 's/nas/n/g'`
    export MYSHORTNAME_IB0=`echo $MYHOSTNAME_IB0   | sed 's/compute/c/g' | sed 's/login/l/g' | sed 's/nas/n/g'`
 
    #
    # Network
    rocks set host interface ip $MYHOSTNAME_ETH0 iface=ib0 ip=$MYHOSTIP_IB0
    rocks set host interface subnet $MYHOSTNAME_ETH0 iface=ib0 subnet=ibnet
    rocks set host interface module $MYHOSTNAME_ETH0 iface=ib0 module=ip_ipoib
    rocks set host interface name $MYHOSTNAME_ETH0 iface=ib0 name=$MYHOSTNAME_ETH0
    rocks sync host network $MYHOSTNAME_ETH0
 
    #
    # Firewall
    rocks add firewall host=$MYHOSTNAME_ETH0 chain=INPUT protocol=all service=all action=ACCEPT network=ibnet iface=ib0 rulename="A80-IB0-PRIVATE"
    rocks sync host firewall $MYHOSTNAME_ETH0
 
    #
    # For debugging purposes only
    printf "%-14s  %-20s  %-14s  %-18s  %-10s\n"  "${MYHOSTIP_ETH0}" "${MYHOSTNAME_ETH0}.local" "${MYSHORTNAME_ETH0}.local" "${MYHOSTNAME_ETH0}" "${MYSHORTNAME_ETH0}"
    printf "%-14s  %-20s  %-14s  %-18s  %-10s\n"  "${MYHOSTIP_IB0}"  "${MYHOSTNAME_ETH0}.ibnet" "${MYSHORTNAME_ETH0}.ibnet" "${MYHOSTNAME_IB0}"  "${MYSHORTNAME_IB0}"
 
    #
    # /etc/hosts.local
    printf "%-14s  %-20s  %-14s  %-18s  %-10s\n"  "${MYHOSTIP_ETH0}" "${MYHOSTNAME_ETH0}.local" "${MYSHORTNAME_ETH0}.local" "${MYHOSTNAME_ETH0}" "${MYSHORTNAME_ETH0}" >> /etc/hosts.local
    printf "%-14s  %-20s  %-14s  %-18s  %-10s\n"  "${MYHOSTIP_IB0}"  "${MYHOSTNAME_ETH0}.ibnet" "${MYSHORTNAME_ETH0}.ibnet" "${MYHOSTNAME_IB0}"  "${MYSHORTNAME_IB0}"  >> /etc/hosts.local
 
  done
  # Inner for loop ends
 
done
# Outer for loop ends

# Front end
10.1.1.1        wigner.local          wigner
10.2.1.1        wigner.ibnet          ib-wigner
# Login node(s)
10.1.255.254    login-0-1.local       l-0-1.local     login-0-1           l-0-1
10.2.255.254    login-0-1.ibnet       l-0-1.ibnet     ib-login-0-1        ib-l-0-1
10.1.255.253    login-0-2.local       l-0-2.local     login-0-2           l-0-2
10.2.255.253    login-0-2.ibnet       l-0-2.ibnet     ib-login-0-2        ib-l-0-2
# NAS node(s)
10.1.255.252    nas-0-0.local         n-0-0.local     nas-0-0             n-0-0
10.2.255.252    nas-0-0.ibnet         n-0-0.ibnet     ib-nas-0-0          ib-n-0-0
# Compute node(s)
10.1.255.251    compute-0-0.local     c-0-0.local     compute-0-0         c-0-0
10.2.255.251    compute-0-0.ibnet     c-0-0.ibnet     ib-compute-0-0      ib-c-0-0
10.1.255.250    compute-0-1.local     c-0-1.local     compute-0-1         c-0-1
10.2.255.250    compute-0-1.ibnet     c-0-1.ibnet     ib-compute-0-1      ib-c-0-1
10.1.255.249    compute-0-2.local     c-0-2.local     compute-0-2         c-0-2
10.2.255.249    compute-0-2.ibnet     c-0-2.ibnet     ib-compute-0-2      ib-c-0-2
10.1.255.248    compute-0-4.local     c-0-4.local     compute-0-4         c-0-4
10.2.255.248    compute-0-4.ibnet     c-0-4.ibnet     ib-compute-0-4      ib-c-0-4