[Kittyhawk] hostnames ending in .0

Jonathan Appavoo jappavoo at bu.edu
Thu Aug 18 13:12:20 EDT 2011


Short answer:

We stupidly compute external ip addresses for both the control server
and other allocatable  compute nodes.  This can result in the ip address
you observer.  In general this is not safe, however, I believe at argonne
this is safe given that the external network (172.16 with a netmask of 255.240).
I believe that it is not right to use a trailing 0, or 255 only in the case of a 24 bit netmask
which we do not have.


Long answer:

Ok this can get kind of messy ;-)

There are three cases to computing ip addresses for the exterior connected nodes:
1) io node bridge addresses
2) khctlserver compute node bridged by the io node
2) other compute nodes bridged by io nodes

in all cases our startup code run at job start time (e.g.  bring up the partition)
sorts out the values.  The code is split between bluegene specific boot 'C' code and 
and uboot scripts code:.  All relevant code for computing addresses is bellow.


 Main points are that on :
A) io nodes we use the address that the site has in it's database via personality page settings.  
B) for khctlserver I need to compute one for each partition that will not conflict but can be known 
     at boot time.  So I have a site specific scheme for hardcoding them which is a kludge
C) For compute nodes we generate addresses based on a mapping from hardware node location to a site specific range of ipaddresses
     in this case: 172 from: ~/Work/kh/Argonne/uboot/board/bluegene/scripts/khctl.argonne.hush



----- code ---



if test $site = "anl-surveyor"
then
  setenv siteprefix 172.16
  setenv sitemask 255.240.0.0  
  setenv sitegate 172.17.3.1
  setenv sitexstart $siteprefix.7.18
  setenv sitexend 172.16.255.255
  
# FIXME: khctleth0ipaddr is inconsistent across nodecards however
#        it probably does not matter as we don't know of dependencies
#        fix to ensure that all nodes see a globally consistent value 
  if itest $bgp_totalnodes > 512
  then
      setenv khctleth0ipaddr "$siteprefix.5.$bgp_nodecard"
  else
    if test $bgp_midplane = "0"
    then
      setenv khctleth0ipaddr "$siteprefix.5.$bgp_nodecard"
    else
      setenv khctleth0ipaddr "$siteprefix.6.$bgp_nodecard"
    fi
  fi
  setenv khctleth0mask $sitemask
  setenv khctlgateway  $sitegate
  setenv khctlsiteconfig 1
fi

if itest $bgp_isio == 1
then
   setenv setenv doboot run boot
   setenv doionode "setenv bootargs console=bgcons khbr=:::$ioeth0ipaddr:$ioeth0mask:$sitemtu init=/init; echo doionode done"
   fdt set /plb/ethernet at 0 tree-channel "[0 0 0 1]"
else
  if itest $bgp_rank == 0 
  then
     mketh 2 1
     mketh 1 0 $sitemtu
     fdt print /plb/ethernet at 0
     setenv ramfsarg $ramfsaddr
     setenv bootargs console=bgcons init=/init ip=$khctleth0ipaddr:$serverip:$khctlgateway:$khctleth0mask::eth0:off khbr=off
     setenv eth0 "$khctleth0ipaddr $khctleth0mask"   
     setenv eth1 "$khctleth1ipaddr $khctleth1mask"
     setenv doramdisk run boot
     unsilence
     echo "KHCTL : setup for $site : khctl @ $khctleth0ipaddr netmask $khctleth0mask gateway $khctlgateway"
     silence
  else
     setenv bgtty_dest 0
     setenv doramdisk
  fi
fi

if itest $bgp_isio == 0 && itest $demoenv == 1
then
   # A default external ipaddress for you
   uni2ip xip $uni $sitexstart $sitexend || echo "ERROR: Failed to set xip from $uni $sitexstart $sitexend"
   setenv xmask $sitemask
   setenv xgate $sitegate
   setenv xboot 'setenv ramfsarg $ramfsaddr && setenv bootargs console=bgtty0,$bgtty_sendid,$bgtty_rcvid,$bgtty_dest init=/init ip=$xip:$serverip:$xgate:$xmask::eth0:off khbr=off'

   # A default internal public ip address for you
   setenv iip $ipaddr
   setenv imask $netmask
   setenv igate $gatewayip
   setenv iboot 'setenv ramfsarg $ramfsaddr && setenv bootargs console=bgtty0,$bgtty_sendid,$bgtty_rcvid,$bgtty_dest init=/init ip=$iip:$serverip:$igate:$imask::eth0:off khbr=off'

   # A some default private network addresses for you
   uni2ip p0ip $bgp_rank 192.168.0.2 192.168.31.255
   setenv p0mask 255.255.224.0
   setenv p0gate 192.168.0.1
   setenv p0boot 'setenv ramfsarg $ramfsaddr && setenv bootargs console=bgtty0,$bgtty_sendid,$bgtty_rcvid,$bgtty_dest init=/init ip=$p0ip:$serverip:$p0gate:$p0mask::eth0:off khbr=off'

   uni2ip p1ip $bgp_rank 192.168.32.2 192.168.63.255
   setenv p1mask 255.255.224.0
   setenv p1gate 192.168.32.1
   setenv p1boot 'setenv ramfsarg $ramfsaddr && setenv bootargs console=bgtty0,$bgtty_sendid,$bgtty_rcvid,$bgtty_dest init=/init ip=$p1ip:$serverip:$p1gate:$p1mask::eth0:off khbr=off'

   uni2ip p2ip $bgp_rank 192.168.64.2 192.168.95.255
   setenv p2mask 255.255.224.0
   setenv p2gate 192.168.64.1
   setenv p2boot 'setenv ramfsarg $ramfsaddr && setenv bootargs console=bgtty0,$bgtty_sendid,$bgtty_rcvid,$bgtty_dest init=/init ip=$p2ip:$serverip:$p2gate:$p2mask::eth0:off khbr=off'

   uni2ip p3ip $bgp_rank 192.168.96.2 192.168.127.255
   setenv p3mask 255.255.224.0
   setenv p3gate 192.168.96.1
   setenv p3boot 'setenv ramfsarg $ramfsaddr && setenv bootargs console=bgtty0,$bgtty_sendid,$bgtty_rcvid,$bgtty_dest init=/init ip=$p3ip:$serverip:$p3gate:$p3mask::eth0:off khbr=off'

   uni2ip p4ip $bgp_rank 192.168.128.2 192.168.159.255
   setenv p4mask 255.255.244.0
   setenv p4gate 192.168.128.1
   setenv p4boot 'setenv ramfsarg $ramfsaddr && setenv bootargs console=bgtty0,$bgtty_sendid,$bgtty_rcvid,$bgtty_dest init=/init ip=$p4ip:$serverip:$p4gate:$p4mask::eth0:off khbr=off'

   setenv dosetxtmpboot 'test -n $bg_eth0_netid && test $bg_eth0_netid = "1" && setenv eth0ip $xip && setenv tmpboot "run xboot && run boot"'
   setenv dosetitmpboot 'test -n $bg_eth0_netid && test $bg_eth0_netid = "2" && setenv eth0ip $iip && setenv tmpboot "run iboot && run boot"'
   setenv dosetptmpboot 'test -n $bg_eth0_netid && setenv eth0ip $p0ip && setenv tmpboot "run p0boot && run boot"' 
   setenv dosettmpboot "run dosetxtmpboot || run dosetitmpboot || run dosetptmpboot"   

   setenv kboot 'run dosettmpboot; test -n $tmpboot && run tmpboot'
fi

from uboot/board/bluegene/bgp:

 if (ccuci->ComputeCard == 0 || ccuci->ComputeCard == 1) {
        setenv("bgp_isio", "1");
        sprintf(tmp, "%d.%d.%d.%d",
                eth->ip.octet[12], eth->ip.octet[13], eth->ip.octet[14],
                eth->ip.octet[15]);
        setenv("bgp_ioeth_ip", tmp);
        
        sprintf(tmp, "%d.%d.%d.%d",
                eth->netmask.octet[12], eth->netmask.octet[13],
                eth->netmask.octet[14], eth->netmask.octet[15]);
        setenv("bgp_ioeth_netmask", tmp);
        
        sprintf(tmp, "%d.%d.%d.%d",
                eth->broadcast.octet[12], eth->broadcast.octet[13],
                eth->broadcast.octet[14], eth->broadcast.octet[15]);
        setenv("bgp_ioeth_broadcast", tmp);
        
        sprintf(tmp, "%d.%d.%d.%d",
                eth->gateway.octet[12],eth->gateway.octet[13],
                eth->gateway.octet[14],eth->gateway.octet[15]);
        setenv("bgp_ioeth_gateway", tmp);
        
        sprintf(tmp, "%d.%d.%d.%d",
                eth->nfsserver.octet[12],eth->nfsserver.octet[13],
                eth->nfsserver.octet[14],eth->nfsserver.octet[15]);
        setenv("bgp_ioeth_nfsserver", tmp);
        
        sprintf(tmp, "%d.%d.%d.%d",
                eth->servicenode.octet[12],eth->servicenode.octet[13],
                eth->servicenode.octet[14],eth->servicenode.octet[15]);
        setenv("bgp_ioeth_servicenode", tmp);
    } else{
        setenv("bgp_isio", "0");
    }

From boot/board/bluegene/bgp/uni.c

unsigned int const COMPUTE_CARD = 34;
unsigned int const NODE_CARD = 16;
unsigned int const MID_PLANE = 2;
unsigned int const RACK_ROW = 16;
unsigned int const RACK_COL = 16;

/* The UCI is presented as the following integer vector:
 *
 * <rack-column, rack-row, mid-plane, node-card, compute-card>
 *
 * We relabel the vector as follows:
 *
 * <v, w, x, y, z>
 *
 * To obtain a monotonically-increasing map from such a vector to 
 * the integers we construct a polynomial of the form:
 *
 * j = Vv + Ww + Xx + Yy + z
 *
 * The coefficients are constructed recursively by adding 1 to
 * the maximum value of the sum of the terms to their right.
 *
 * Y = 1 + max(z)
 * X = 1 + max(z) + max(Yy)
 * W = 1 + max(z) + max(Yy) + max (Xx)
 * V = 1 + max(z) + max(Yy) + max (Xx) + max(Ww)
 *
 * This can be simplified:
 *
 * Y = 1 + 1 * max(z)
 * X = Y + Y * max(y)
 * W = X + X * max(x)
 * V = W + W * max(w)
 *
 * Note that the range 1-2 is a hole for compute-card.
 */

unsigned int bg_uci_to_uni(BGP_UCI_ComputeCard_t *uci)
{
    unsigned int j;
    unsigned int v, w, x, y, z;
    unsigned int V, W, X, Y;

    Y = 1 + 1 * (COMPUTE_CARD - 1);
    X = Y + Y * (NODE_CARD - 1);
    W = X + X * (MID_PLANE - 1);
    V = W + W * (RACK_ROW - 1);

    v = uci->RackColumn;
    w = uci->RackRow;
    x = uci->Midplane;
    y = uci->NodeCard;
    z = uci->ComputeCard >= 4 ? uci->ComputeCard - 2 : uci->ComputeCard;

    j = V*v + W*w + X*x + Y*y + z;

    return j;
}

unsigned int bg_uni_to_ipv4(unsigned int uni,
                            unsigned char *oc3, 
                            unsigned char *oc2, 
                            unsigned char *oc1, 
                            unsigned char *oc0,
                            unsigned char start_oc3,
                            unsigned char start_oc2,
                            unsigned char start_oc1,
                            unsigned char start_oc0,
                            unsigned char end_oc3,
                            unsigned char end_oc2,
                            unsigned char end_oc1,
                            unsigned char end_oc0)
{
    unsigned int start, end, ip;
    
    start = start_oc3 << 24 | start_oc2 << 16 | start_oc1 << 8 | start_oc0;
    end   = end_oc3 << 24 | end_oc2 << 16 | end_oc1 << 8 | end_oc0;

    ip = start + uni;

    if (ip > end || ip < start) return 0;
    
    *oc0 = ip & 0xff;
    *oc1 = (ip >> 8) & 0xff;
    *oc2 = (ip >> 16) & 0xff;
    *oc3 = (ip >> 24) & 0xff;
#if 0
    printf("uni=%d(0x%x) start=%d(0x%x) end=%d(0x%x) ip=%d(0x%x) oc3=%d(0x%x)"
           " oc2=%d(0x%x) oc1=%d(0x%x) oc0=%d(0x%x)\n", uni,uni,
           start, start, end, end,ip,ip, *oc3, *oc3, *oc2, *oc2, *oc1, *oc1, *oc0, *oc0);
#endif
    return 1;
}


On Aug 17, 2011, at 6:14 PM, ron minnich wrote:

> I don't know how KH generates this type of hostname: 172.16.5.0
> 
> it works, which is always a surprise, but it is actually illegal in
> most contexts I work in, since it's usually a network number.
> 
> But maybe there is something I don't understand ....
> 
> ron
> _______________________________________________
> Kittyhawk mailing list
> Kittyhawk at cs.bu.edu
> http://cs-mailman.bu.edu/mailman/listinfo/kittyhawk

-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://cs-mailman.bu.edu/pipermail/kittyhawk/attachments/20110818/a9382b62/attachment-0001.html 


More information about the Kittyhawk mailing list