NetworkStack important structure

Version: linux-kernel-1.2.13

File                Struct                        Description
-------------------------------------------------------------------
path: /include/linux
etherdevice.h                                     以太网协议相关函数声明
icmp.h                                            ICMP协议结构定议
                    icmphdr
                    icmp_err
if.h                                              接口相关结构定议
                    ifmap
                    ifreq
                    ifconf
if_arp.h                                          ARP协议结构定议
                    arpreq
                    arphdr
if_ether.h                                        以太网首部及标志位定义
                    ethhdr
                    enet_statistics
if_plip.h                                         并行线网络协议
                    plip_conf
if_slip.h                                         串行线协议
igmp.h                                            IGMP协议结构定义
                    igmphdr
                    ip_mc_socklist
                    ip_mc_list
in.h                                              协议号定义
                    in_addr
                    ip_mreq
                    sockaddr_in
inet.h                                            INET域部分函数声明
interrupt.h                                       下半部分结构定义
                    bh_struct
ip.h                                              IP协议结构定义
                    timestamp
                    route
                    iphdr
ip_fw.h                                           防火墙相关结构定义
                    ip_fw
                    ip_fwpkt
ipx.h                                             IPX包交换协议结构定义
                    sockaddr_ipx
                    ipx_route_definition
                    ipx_interface_definition
                    ipx_config_data
                    ipx_route_def
net.h                                             INET层关键结构定义
                    socket
                    proto_ops
                    net_proto
netdevice.h                                       设备相关结构定义
                    de_mc_list
                    device
                    packet_type
notifier.h                                        事件响应相关结构定义
                    notifier_block
ppp.h                                             点对点协议结构定义
                    ppp_lqp_packet_hdr
                    ppp_lqp_packet_trailer
                    ppp_lqp_packet
                    ppp_ddinfo
                    ppp
route.h                                           路由结构定义
                    old_rtentry
                    rtentry
skbuff.h                                          数据包封装结构定义
                    sk_buff_head
                    sk_buff
socket.h                                          常数选项定义
                    sockaddr
                    linger
sockios.h                                         选项定义
tcp.h                                             TCP协议结构定义
                    tcphdr
timer.h                                           定时器相关结构定义
                    timer_struct
                    timer_list
udp.h                                             UDP协议结构定义
                    udphdr
un.h                                              UNIX域地址结构定义
                    sockaddr_un

path: net/inet
ip.h
                    ipfraq
                    ipq 
ipx.h
                    ipx_address
                    ipx_packet
                    ipx_interface
                    ipx_route
protocol.h
                    inet_protocol
route.h
                    rtable
snmp.h
                    ip_mib
                    icmp_mib
                    tcp_mib
                    udp_mib
sock.h
                    sock
                    proto

path: include/linux
fs.h
                    file
                    inode
                    file_operations
                    inode_operations

Structure and memberships

+---------------------------+              +------------------------+
|struct sk_buff             |              |struct sk_buff_head     |
+---------------------------+              +------------------------+
|struct sk_buff    *next;      |              |struct sk_buff  *next;  |
|struct sk_buff    *prev;      |              |struct sk_buff  *prev;  |
|struct sk_buff    *link3;     |              +------------------------+
|struct sk_buff *mem_addr;  |              
|struct sock *sk;           |              +--------------------------+ 
|struct device *dev;        |              |struct proto              |
|... ...                    |              +--------------------------+
+---------------------------+              |struct sock *sock_array[];|
                                           |int (*init) ();           |
                                           |int (*setsockopt) ();     |
+----------------------------+             |int (*read) ();           |
|struct device               |             |int (*write) ();          |
+----------------------------+             |int (*connect) ();        |
|struct device *next;        |             |int (*sendto) ();         |
|struct device *slave;       |             |int (*recvfrom)();        |
|struct sk_buff_head buffs[];|             |... ...                   |
|... ...                     |             +--------------------------+
+----------------------------+             

+----------------------------------+       +----------------------------------+
|struct socket                     |       |struct sock                       |
+----------------------------------+       +----------------------------------+
|short                type;        |       |struct sock *next;                |
|socket_state         state;       |       |struct sk_buff_head write_queue;  |
|long                 flags;       |       |struct sk_buff_head receive_queue;|
|struct proto_ops     *ops;        |       |struct proto *prot;               |
|void                 *data;       |       |struct socket *socket;            |
|struct socket        *conn;       |       |struct sk_buff *send_head;        |
|struct socket        *iconn;      |       |struct sk_buff *send_tail;        |
|struct socket        *next;       |       |struct sk_buff_head back_log;     |
|struct wait_queue    **wait;      |       |struct sk_buff *partial;          |
|struct inode         *inode;      |       |... ...                           |
|struct fasync_struct *fasync_list;|       +----------------------------------+
|... ...                           |       
+----------------------------------+       
                                           +-----------------------------+
+-----------------------+                  |struct file                  |
|struct proto_ops       |                  +-----------------------------+
+-----------------------+                  |struct file *f_next, *f_prev;|
|int    family();       |                  |struct inode *f_inode;       |
|int    (*create)();    |                  |struct file_operations *f_op;|
|int    (*dup)();       |                  |... ...                      |
|int    (*release)();   |                  +-----------------------------+
|int    (*bind)();      |                  
|int    (*connect)();   |                  +-----------------------------+
|int    (*socketpair)();|                  |struct file_operations       |
|int    (*accept)();    |                  +-----------------------------+
|int    (*getname)();   |                  |int (*open) ();              |
|int    (*read)();      |                  |int (*read) ();              |
|int    (*write)();     |                  |int (*write) ();             |
|int    (*select)();    |                  |int (*select) ();            |
|int    (*ioctl)();     |                  |int (*release) ();           |
|int    (*listen)();    |                  |... ...                      |
|int    (*send)();      |                  +-----------------------------+
|int    (*recv)();      |
|int    (*sendto)();    |                  +------------------------------+
|int    (*recvfrom)();  |                  |struct inode                  |
|int    (*shutdown)();  |                  +------------------------------+
|int    (*setsockopt)();|                  |struct inode *i_next, *i_prev;|
|int    (*getsockopt)();|                  |struct inode_operations *i_op;|
|int    (*fcntl)();     |                  |... ...                       |
+-----------------------+                  +------------------------------+

NetworkStack Internet protocol suite

1. Introduce

The Internet protocol suite and the layered protocol stack design were in use
before the OSI model was established. Since then, the TCP/IP model has been
compared with the OSI model in books and classrooms, which often results in
confusion because the two models use different assumptions and goals, including
the relative importance of strict layering.

2. TCP/IP Network model

Layers in the internet protocol suite:( a four-layer model )

+------------------------+                         +--------+
|    Application layer   |                         |  Data  |
+------------------------+                         +--------+
            |
  +--------------------+                   +-------+--------+
  |   Transport layer  |                   |TCP/IP |  Data  |
  +--------------------+                   +-------+--------+
            |
  +-------------------+              +-----+-------+--------+
  |   Internet layer  |              | IP  |TCP/IP |  Data  |
  +-------------------+              +-----+-------+--------+
            |
    +---------------+       +--------+-----+-------+--------+
    |   Link layer  |       | Header | IP  |TCP/IP |  Data  |
    +---------------+       +--------+-----+-------+--------+

Protocol in layers

Application layer

DHCP
DHCPv6
DNS
FTP
HTTP*
IMAP
IRC
LDAP
MGCP
NNTP
BGP
NTP
POP
RPC
RTP
RTSP
RIP
SIP
SMTP
SNMP*
SOCKS
SSH
Telnet
TLS/SSL
XMPP

Transport layer

TCP*
UDP*
DCCP
SCTP
RSVP

Internet layer

IP(IPv4,IPv6)*
ICMP*
ICMPv6
ECN
IGMP
IPsec

Link layer

ARP/InARP*
NDP
OSPF
Tunnels(L2TP)
PPP*
Media access control(Ethernet/DSL/ISDN/FDDI)

3. Protocol Format

IP - http://www.ietf.org/rfc/rfc791.txt, page 11
TCP - http://www.ietf.org/rfc/rfc793.txt, page 15
UDP - http://www.ietf.org/rfc/rfc768.txt, page 1

0                   1                   2                   3   
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|Version|  IHL  |Type of Service|          Total Length         |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|         Identification        |Flags|      Fragment Offset    |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|  Time to Live |    Protocol   |         Header Checksum       |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                       Source Address                          |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                    Destination Address                        |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                    Options                    |    Padding    |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

              Example Internet Datagram Header

0                   1                   2                   3   
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|          Source Port          |       Destination Port        |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                        Sequence Number                        |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                    Acknowledgment Number                      |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|  Data |           |U|A|P|R|S|F|                               |
| Offset| Reserved  |R|C|S|S|Y|I|            Window             |
|       |           |G|K|H|T|N|N|                               |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|           Checksum            |         Urgent Pointer        |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                    Options                    |    Padding    |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|                             data                              |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

                        TCP Header Format

0      7 8     15 16    23 24    31  
+--------+--------+--------+--------+ 
|     Source      |   Destination   | 
|      Port       |      Port       | 
+--------+--------+--------+--------+ 
|                 |                 | 
|     Length      |    Checksum     | 
+--------+--------+--------+--------+ 
|                                     
|          data octets ...            
+---------------- ...                 

     User Datagram Header Format

IP - Internet Protocol
http://www.ietf.org/rfc/rfc791.txt

ARP - Address Resolution Protocol
http://www.ietf.org/rfc/rfc826.txt
http://en.wikipedia.org/wiki/Arp_protocol

ICMP - Internet Control Message Protoco
http://en.wikipedia.org/wiki/Internet_Control_Message_Protocol
http://www.ietf.org/rfc/rfc792.txt

TCP - Transmission Control Protocol
http://en.wikipedia.org/wiki/Transmission_Control_Protocol
http://www.ietf.org/rfc/rfc793.txt

SMTP - Simple Mail Transfer Protocol
http://www.ietf.org/rfc/rfc821.txt

FTP - File Transfer Protocol
http://www.ietf.org/rfc/rfc958.txt

SNMP - Simple Network Management Protocol
http://www.ietf.org/rfc/rfc1067.txt

IGMP - Internet Group Management Protocol
http://www.ietf.org/rfc/rfc3376.txt
http://en.wikipedia.org/wiki/Internet_Group_Management_Protocol

UDP - User Datagram Protocol
http://en.wikipedia.org/wiki/User_Datagram_Protocol
http://www.ietf.org/rfc/rfc768.txt

PPP - Point to point protocol
http://en.wikipedia.org/wiki/Point-to-point_protocol
http://tools.ietf.org/html/rfc1661

SMTP - Simple Mail Transfer Protocol
http://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol

HTTP - Hypertext Transfer Protocol
http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol

5. Reference

MAC address swap with two network cards

1. Problem

1.1 When network setting is normal, found that MAC address was swapped.

eth0 –> 00:25:90:70:51:f1
eth1 –> 00:25:90:70:51:f0

[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:25:90:70:51:f1 brd ff:ff:ff:ff:ff:ff
    inet 172.16.110.230/24 brd 172.16.110.255 scope global eth0
    inet6 fe80::225:90ff:fe70:51f1/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:25:90:70:51:f0 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::225:90ff:fe70:51f0/64 scope link 
       valid_lft forever preferred_lft forever

1.2 But when made a bind to the two interface, it was correct.

[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master bond0 state UP qlen 1000
    link/ether 00:25:90:70:51:f0 brd ff:ff:ff:ff:ff:ff
3: eth1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master bond0 state UP qlen 1000
    link/ether 00:25:90:70:51:f1 brd ff:ff:ff:ff:ff:ff
4: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:25:90:70:51:f1 brd ff:ff:ff:ff:ff:ff
    inet 172.16.110.230/24 brd 172.16.110.255 scope global bond0
    inet6 fe80::225:90ff:fe70:51f1/64 scope link 
       valid_lft forever preferred_lft forever

What had done when making the bind?

Use command dmesg, we found that if there were words like “udev: renamed XXX to XXX”,
the MAC addree was correct matched to the interface. If not it was error.

before changed:

[root@localhost ~]# dmesg | grep "PCI Express"
pciehp: PCI Express Hot Plug Controller Driver version: 0.4
e1000e 0000:00:19.0: eth0: (PCI Express:2.5GT/s:Width x1) 00:25:90:70:51:f1
e1000e 0000:04:00.0: eth1: (PCI Express:2.5GT/s:Width x1) 00:25:90:70:51:f0

[root@localhost ~]# dmesg | grep udev
udev: starting version 147
udev: starting version 147

[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE="eth0"
BOOTPROTO="dhcp"
HWADDR="00:25:90:70:51:F1"
NM_CONTROLLED="yes"
ONBOOT="no"
TYPE="Ethernet"
UUID="151c6203-2172-41b1-bd58-9bf03a2731e7"

[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
DEVICE="eth1"
BOOTPROTO="dhcp"
HWADDR="00:25:90:70:51:F0"
NM_CONTROLLED="yes"
ONBOOT="no"
TYPE="Ethernet"
UUID="a677dbc6-144a-455b-bcd3-6d8439560516"

[root@localhost ~]# cat /etc/udev/rules.d/70-persistent-net.rules 
# This file was automatically generated by the /lib/udev/write_net_rules
# program, run by the persistent-net-generator.rules rules file.
#
# You can modify it, as long as you keep each rule on a single
# line, and change only the value of the NAME= key.

# PCI device 0x8086:0x10d3 (e1000e) (custom name provided by external tool)
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:25:90:70:51:f0", ATTR{type}=="1", KERNEL=="eth*", NAME="eth1"

# PCI device 0x8086:0x1502 (e1000e) (custom name provided by external tool)
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:25:90:70:51:f1", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0"

after changed:

[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE="eth0"
BOOTPROTO="dhcp"
HWADDR="00:25:90:70:51:F0"
NM_CONTROLLED="yes"
ONBOOT="no"
TYPE="Ethernet"
UUID="a677dbc6-144a-455b-bcd3-6d8439560516"
[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
DEVICE="eth1"
BOOTPROTO="dhcp"
HWADDR="00:25:90:70:51:F1"
NM_CONTROLLED="yes"
ONBOOT="no"
TYPE="Ethernet"
UUID="151c6203-2172-41b1-bd58-9bf03a2731e7"

then reboot system.

[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:25:90:70:51:f1 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::225:90ff:fe70:51f1/64 scope link 
       valid_lft forever preferred_lft forever
3: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:25:90:70:51:f0 brd ff:ff:ff:ff:ff:ff
    inet 172.16.110.230/24 scope global eth0
    inet6 fe80::225:90ff:fe70:51f0/64 scope link 
       valid_lft forever preferred_lft forever

[root@localhost ~]# dmesg | grep udev
udev: starting version 147
udev: starting version 147
udev: renamed network interface eth0 to rename2
udev: renamed network interface eth1 to eth0
udev: renamed network interface rename2 to eth1

[root@localhost ~]# cat /etc/udev/rules.d/70-persistent-net.rules 
# This file was automatically generated by the /lib/udev/write_net_rules
# program, run by the persistent-net-generator.rules rules file.
#
# You can modify it, as long as you keep each rule on a single
# line, and change only the value of the NAME= key.

# PCI device 0x8086:0x10d3 (e1000e) (custom name provided by external tool)
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:25:90:70:51:f0", ATTR{type}=="1", KERNEL=="eth*", NAME="eth1"

# PCI device 0x8086:0x1502 (e1000e) (custom name provided by external tool)
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:25:90:70:51:f1", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0"

# PCI device 0x8086:0x1502 (e1000e) (custom name provided by external tool)
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:25:90:70:51:f1", ATTR{type}=="1", KERNEL=="eth*", NAME="eth1"

# PCI device 0x8086:0x10d3 (e1000e) (custom name provided by external tool)
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:25:90:70:51:f0", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0"

[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE="eth0"
BOOTPROTO="dhcp"
HWADDR="00:25:90:70:51:F0"
NM_CONTROLLED="yes"
ONBOOT="no"
TYPE="Ethernet"
UUID="a677dbc6-144a-455b-bcd3-6d8439560516"
[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
DEVICE="eth1"
BOOTPROTO="dhcp"
HWADDR="00:25:90:70:51:F1"
NM_CONTROLLED="yes"
ONBOOT="no"
TYPE="Ethernet"
UUID="151c6203-2172-41b1-bd58-9bf03a2731e7"
[root@localhost ~]#

1.3 make a bind to the network, MAC is correct also.

[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master bond0 state UP qlen 1000
    link/ether 00:25:90:70:51:f1 brd ff:ff:ff:ff:ff:ff
3: eth0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master bond0 state UP qlen 1000
    link/ether 00:25:90:70:51:f0 brd ff:ff:ff:ff:ff:ff
4: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:25:90:70:51:f0 brd ff:ff:ff:ff:ff:ff
    inet 172.16.110.230/24 brd 172.16.110.255 scope global bond0
    inet6 fe80::225:90ff:fe70:51f0/64 scope link 
       valid_lft forever preferred_lft forever
[root@localhost ~]# 

delete the bind, still ok.

1.4 How for group ip?

group set to 172.16.110.233
eth0 set to 172.16.110.230
eth1 set to 172.16.110.231
from below we found the MAC was ok as hoped.

[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:25:90:70:51:f1 brd ff:ff:ff:ff:ff:ff
    inet 172.16.110.231/24 scope global eth1
    inet 172.16.110.233/24 scope global secondary eth1
    inet6 fe80::225:90ff:fe70:51f1/64 scope link 
       valid_lft forever preferred_lft forever
3: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:25:90:70:51:f0 brd ff:ff:ff:ff:ff:ff
    inet 172.16.110.230/24 scope global eth0
    inet 172.16.110.233/24 scope global secondary eth0
    inet6 fe80::225:90ff:fe70:51f0/64 scope link 
       valid_lft forever preferred_lft forever

2. The steps to modify MAC of eth* look like as below

  1. OS boot
  2. udev start
  3. e1000e start, get eth* and MAC
  4. check /etc/sysconfig/network-scripts/ifcfg-XXX, call shell script
    /lib/udev/write_net_rules to generate rule file to /etc/udev/rules.d/70-persistent-net.rules
    if MAC was not match with what got from e1000e (view udev source, you wil find udev)
  5. udev rename eth* if there have “add” action in rule file /etc/udev/rules.d/70-persistent-net.rules

2.1 Check /lib/udev/write_net_rules

36 RULES_FILE='/etc/udev/rules.d/70-persistent-net.rules'
37 
38 . /lib/udev/rule_generator.functions
39 
40 find_all_ifcfg() {
41     local links=$1
42     local __sed_discard_ignored_files='/\(~\|\.bak\|\.orig\|\.rpmnew\|\.rpmorig\|\.rpmsave\)$/d'
43 
44     files=$(echo /etc/sysconfig/network-scripts/ifcfg-* \
45         | LC_ALL=C sed -e "$__sed_discard_ignored_files")
46     for i in $files; do
47         (
48             . $i
49             [ -n "$HWADDR" ] && [ "${links%%[ \[\]0-9]*}" = "${DEVICE%%[ \[\]0-9]*}" ] && echo $DEVICE
50         )
51     done
52 }

70 write_rule() {
71         local match="$1"
72         local name="$2"
73         local comment="$3"
74 
75         {
76         if [ "$PRINT_HEADER" ]; then
77                 PRINT_HEADER=
78                 echo "# This file was automatically generated by the $0"
79                 echo "# program, run by the persistent-net-generator.rules rules file."
80                 echo "#"
81                 echo "# You can modify it, as long as you keep each rule on a single"
82                 echo "# line, and change only the value of the NAME= key."
83         fi
84 
85         echo ""
86         [ "$comment" ] && echo "# $comment"
87         echo "SUBSYSTEM==\"net\", ACTION==\"add\"$match, NAME=\"$name\""
88         } >> $RULES_FILE
89 }

153 write_rule "$match" "$INTERFACE" "$COMMENT"

2.1 Check udev source

search the string we found in dmesg:

[dennis@localhost udev]$ pwd
/home/dennis/Downloads/udev-174/udev
[dennis@localhost udev]$ grep -r -i --include "*.c" "renamed" .
./udev-rules.c:    /* handle device, renamed by external tool, returning new path */
./udev-event.c:    fprintf(f, "<30>udevd[%u]: renamed network interface %s to %s\n",
./udev-event.c:                info(event->udev, "renamed netif to '%s'\n", event->name);
[dennis@localhost udev]$ gvim udev-event.c 

static void rename_netif_kernel_log(struct ifreq ifr)
{
    int klog;
    FILE *f;

    klog = open("/dev/kmsg", O_WRONLY);
    if (klog < 0)
        return;

    f = fdopen(klog, "w");
    if (f == NULL) {
        close(klog);
        return;
    }

    fprintf(f, "<30>udevd[%u]: renamed network interface %s to %s\n",
        getpid(), ifr.ifr_name, ifr.ifr_newname);
    fclose(f);
}

Because udev write log to /dev/kmsg, so we can got output like below:

[root@localhost ~]# dmesg | grep udev
udev: starting version 147
udev: starting version 147
udev: renamed network interface eth0 to rename2
udev: renamed network interface eth1 to eth0
udev: renamed network interface rename2 to eth1

Continue tracking, found that rename_netif_kernel_log was called only by function:

static int rename_netif(struct udev_event *event)

And who call rename_netif, the answer is only function:

int udev_event_execute_rules(struct udev_event *event, struct udev_rules *rules, const sigset_t *sigmask)

From code segment from this function, we see:

....
/* rename a new network interface, if needed */
if (udev_device_get_ifindex(dev) > 0 && strcmp(udev_device_get_action(dev), "add") == 0 &&
    event->name != NULL && strcmp(event->name, udev_device_get_sysname(dev)) != 0) {
    char syspath[UTIL_PATH_SIZE];
    char *pos;

    err = rename_netif(event);
....

from the name of function udev_event_execute_rules and cod segment, we can say,
udev rename ethXXX by the rules, where should have action like as “add”.

2.2 Check e1000e source, found nothing

2.3 Check dmesg source, found nothing

2.4 Check system-config-network source, found nothing

3. How to fix

Use this shell scripts to check MAC

#!/bin/sh

# check MAC address, if not match the correct interface, fix it.
file_eth0=/etc/sysconfig/network-scripts/ifcfg-eth0
file_eth1=/etc/sysconfig/network-scripts/ifcfg-eth1

if [ -e $file_eth0 ] && [ -e $file_eth1 ]; then 
    # get MAC and UUID
    MAC0=$(grep HWADDR $file_eth0 | awk -F'"' '{print $(NF-1)}')
    MAC1=$(grep HWADDR $file_eth1 | awk -F'"' '{print $(NF-1)}')
    UUID0=$(grep UUID  $file_eth0 | awk -F'"' '{print $(NF-1)}')
    UUID1=$(grep UUID  $file_eth1 | awk -F'"' '{print $(NF-1)}')

    if [ "$MAC0" != "" ] && [ "$MAC1" != "" ]; then
        # compare MAC
        compare_flag=0
        compare_flag=`echo $MAC0 $MAC1 | awk '$1>$2 {print 1}'`
        if [ "$compare_flag" == "1" ]; then
            # swap MAC and UUID
            sed -i 's/'$MAC0'/'$MAC1'/g' $file_eth0
            sed -i 's/'$MAC1'/'$MAC0'/g' $file_eth1
            if [ "$UUID0" != "" ] && [ "$UUID1" != "" ]; then
                sed -i 's/'$UUID0'/'$UUID1'/g' $file_eth0
                sed -i 's/'$UUID1'/'$UUID0'/g' $file_eth1
            else
                sed -i '/UUID/d' $file_eth0
                [ "$UUID1" != "" ] && echo "UUID=\"$UUID1\"" >> $file_eth0
                sed -i '/UUID/d' $file_eth1
                [ "$UUID0" != "" ] && echo "UUID=\"$UUID0\"" >> $file_eth1
            fi
        fi
    fi
fi

4. Question

  1. /etc/sysconfig/network-scripts/ifcfg-XXX were generated by who? system-config-network?

5. Reference

Steven Jobs: Stanford commencement address, June 2005

From http://english.koolearn.com/20111008/478099.html

This is the text of the Commencement address by Steven Jobs, CEO of Apple Computer and of Pixar Animation Studios, delivered on June 12, 2005.

I am honored to be with you today at your commencement from one of the finest universities in the world. I never graduated from college. Truth be told, I never graduated from college. This is the closest I’ve ever gotten to a college graduation. Today I want to tell you three stories from my life. That’s it. No big deal. Just three stories.

The first story is about connecting the dots.

I dropped out of Reed College after the first 6 months, but then stayed around as a drop-in for another 18 months or so before I really quit. So why did I drop out?

It started before I was born. My biological mother was a young, unwed college graduate student, and she decided to put me up for adoption. She felt very strongly that I should be adopted by college graduates, so everything was all set for me to be adopted at birth by a lawyer and his wife. Except that when I popped out they decided at the last minute that they really wanted a girl. So my parents, who were on a waiting list, got a call in the middle of the night asking: “We have an unexpected baby boy; do you want hime?” They said: “Of course.” My biological mother later found out that my mother had never graduated from college and that my father had never graduated from high school. She refused to sign the final adoption papers. She only relented a few months later when my parents promised that I would someday go to college.

And 17 years later I did go to college. But I naively chose a college that was almost as expensive as Stanford, and all of my working-class parents’ savings were being spent on my college tuition. After six months, I couldn’t see the value in it. I had no idea what I wanted to do with my life and no idea how college was going to help me figure it out. And here I was spending add of the money my parents had saved their entire life. So I decided to drop out and trust that it would all work out OK. It was pretty scary at the time, but looking back it was one of the best decisions I ever made. The minute I dropped out I could stop taking the required classes that didn’t interest me, and begin dropping in on the ones that looked interesting.

It wasn’t all romantic. I didn’t have a dorm room, so I slept on the floor in friends’ rooms. I returned coke bottles for the 5 cent; deposites to buy food with, and I would walk the 7 miles across town every Sunday night to get one good meal a week at the Hare Krishna temple. I loved it. And much of what I stumbled into by following my curiosity and intuition turned out to be priceless later on. Let me give you one example.

Reed College at that time offered perhaps the best calligraphy instruction in the country. Throughout the campus every poster, every label on every drawer, was beautifully hand calligraphed. Because I had dropped out and didn’t have to take the normal classes. I decided to take a calligraphy class to learn how to do this. I learned about serif and san serif typefaces, about varying the amount of space between different letter combinations, about what makes great typography great. It was beautiful, historical, artistically subtle in a way that science can’t capture and I found it fascinating.

None of this had even a hope of any practical application in my life. But ten years later, when we were designing the first Macintosh computer, it all came back to me. And we designed it all into the Mac. It was the first computer with beautiful typography. If I had never dropped in on that single course in college, the Mac would have never had multiple typefaces or proportionally space fonts. And since Windows just copied the Mac, its likely that no personal computer would have them. If I had never dropped out, I would have never dropped in on this calligraphy class, and personal computers might not have the wonderful typography that they do. Of course it was impossible to connect the dots looking forward when I was in college. But it was very, very clear looking backwards ten years later.

Again, you can’t connect the dots looking forward; you can only connect them looking backwards. So you have to trust that the dots will somehow connect in your future. You have to trust in something – your gut, destiny, life, karma, wahtever. This approach has never let me down, and it has made all the difference in my life.

My second story is about love and loss.

I was lucky – I found what I loved to do early in life. Woz and I started Apple in my parents garage when I was 20. We worked hard, and in 10 years Apple had grown from just the two of us in a garage into a $2 billion company with over 4000 employees. We had just released out finest creation – the Macintosh – a year earlier, and I had just turned 30. And then I got fired. How can you get fired from a company you started? Well, as Apple grew we hired someone who I thought was very talented to run the company with me, and for the first year or so things went well. But then our visions of the future began to diverge and eventually we had a falling out. When we did, our Board of Directors sided with him. So at 30 I was out. And very publicly out. What had been the focus of my entire adult life was gone, and it was devastating.

I really didn’t know what to do for a few months. I felt that I had let the previous generation of entrepreneurs down - that I had dropped the baton as it was being passed to me. I met with David Packard and Bob Noyce and tried to apologize for screwing up so badly. I was a very public failure, and I even thought about running away from the valley. But something slowly began to dawn on me – I still loved what I did. The turn of events at Apple had not changed that one bit. I had been rejected, but I was still in love. And so I decided to start over.

I didn’t seen it then, but it turned out that getting fired from Apple was the best thing that could have ever happened to me. The heaviness of being successful was replaced by the lightness of being a beginner again, less sure about everything. It freed me to enter one of the most creative periods of my life.

During the next five years, I started a company named NeXT, another company named Pixar, and fell in love with an amazing woman who would become my wife. Pixar went on to create the worlds first computer animated feature film, Toy Story, and is now the most successful animation studio in the world. In a remarkable turn of events, Apple bought NeXT, I returned to Apple, and the technology we developed at NeXT is at the heart of Apple’s current renaissance. And Laurene and I have a wonderful family together.

I’m pretty sure none of this would have happened if I hadn’t been fired from Apple. It was awful tasting medicine, but I guess the patient needed it.

Sometimes life hits you in the head with a brick. Don’t lose faith. I’m convinced that the only thing that kept me going was that I loved what I did. You’ve got to find what you love. And that is as true for you work as it is for your lovers. Your work is going to fill a large part of your life, and the only way to be truely satisfied is to do what you believe is great work. And the only way to do great work is to love what you do. If you haven’t found it yet, keep looking. Don’t settle. As with all matters of the heart, you’ll know when you find it. And, like any great relationship, it just gets better and better as the years roll on. So keep looking until you find it. Don’t settle.

My third story is about death

When I was 17, I read a quote that went something like: “If you live each day as if it was your last, someday you’ll most certainly be right.” It made an impression on me, and since then, for the past 33 years, I have looked in the mirror every morning and asked myself: “If today were the last day of my life, would I want to do what I am about to do today?” And whenever the answer has been “No” for too many days in a row, I know I need to change something.

Remembering that I’ll be dead soon is the most important tool I’ve ever encountered to help me make the big choices in life. Because almost everything – all external expectations, all pride, all fear of embarrassment or failure - these things just fall away in the face of death, leaving only what is truly important. Remembering that you are going to die is the best way I know to avoid the trap of thinking you have something to lose. You are already naked. There is no reason not to follow your heart.

About a year ago I was diagnosed with cancer. I had a scan at 7:30 in the morning, and it clearly showed a tumor on my pancreas. I didn’t even know what a pancreas was. The doctors told me this was almost certainly a type of cancer that is incurable, and what I should expect to live no longer than three to six months. My doctor advised me to go home and get my affairs in order, which is doctor’s code for prepare to die. It means to try to tell your kids everything you thought you’d had the next 10 years to tell them in just a few months. It means to make sure everything is buttoned up so that it will be easy as possible for your family. It means to say your goodbyes.

I lived with that diagnosis all day. Later that evening I had a biopsy, where they stuck anendoscope down my throat, through my stomach and into my intestines, put a needle into my pancreas and got a few cells from the tumor. I was sedated, but my wife, who was there, told me that when they viewed the cells under a microscope the doctors started crying because it turned out to be a very rare from of pancreatic cancer that is curable with surgery. I had surgery and I’m fine now.

This was the closet I’ve been to facing death, and I hope its the closet I get for a few more decades. Having lived through it, I can now say this to you with a bit more certainty than when death was a useful but purely intellectual concept:

No one wants to die. Even people who want to go to heaven don’t want to die to get there. And yet death is the destination we all share. No one has ever escaped it. And that is as it should be, because Death is very likely the single best invention of Life. It is Life’s change agent. It clears out the old to make way for the new. Right now the new is you, but someday not too long from now, you will gradually become the old and be cleared away. Sorry to be so dramatic, but it is quite true.

Your tiems is limited, so don’t waste it living someone else’s life. Don’t be trapped by dogma – which is living with the results of other people’s thinking. Don’t let the noise of other’s opinions drown out your own inner voice. And most important, have the courage to follow your heart and intuition. They somehow already know what you truly want to become. Everything else is secondary.

When I was young, there was an amazing publication called The Whole Earth Catalog, which was one of the bibles of my generation. It was created by a fellow named Stewart Brand not far from here in Menlo Park, and he brought it to life with his poetic touch. This was in the late 1960’s, before personal computers and desktop publishing, so it was all made with typewriters, scissors, and polaroid cameras. It was sort of like Goolge in paperback form, 35 years before Google came along: it was idealistic, and overflowing with neat tools and great notions.

Stewart and his team put out several issues of the Whole Earth Catalog, and then when it had run its course, they put out a final issue. It was the mid-1970s, and I was your age. On the back cover of their final issue was photograph of an early morning country road, the kind you might find yourself hitchhiking on if you were so adventurous. Beneath it were the words: “Stay Hungry. Stay Foolish.” It was their farewell message as they signed off. Stay Hungry. Stay Foolish. And I have always wished that for myself. And now, as your graduate to begin anew, I wish that for you.

Stay Hungry. Stay Foolish.

Thank you all very much.

novels Three Peaks

From

#Part One

#Everest

##Chapter One

Just outside of Namche Bazaar, Nepal

present day

Sunita was breathing hard.

Not good. The air up here made one light-headed. Nepal had eight of the world’s ten tallest mountain peaks, but right now, Sunita’s thoughts were a lot higher.

“Oh jesus!” she cried, as she kept moving along the narrow path. “Save me! Save me!”

Near breathless, she repeated it in her heart rather than out loud.

If the Maoist Army caught up with her, she could be executed for being an American spy. She should know. She used to be a Lance Corporal in the army. As for being an American spy, her only crime was to have been caught reading a Christian pamphlet that one of her fellow officers had given her.

It told the story of a man who had lived and then died … and then lived again. She had read it through several times. The little pamphlet had now been lost in flight. Which was a shame because she would have liked to know more.

At least there was one person she knew who could tell her more.

But her immediate objective was to survive.

“On, Jesus!” she cried. It was more of a whisper. “Save me!”

Joliet, Illinois

present day

“With awesome deeds you answer us with righteousness, God our salvation, the hope of all the ends of the earth and of the farthest seas.” Chris surveyed his congregation. The knew this scripture by heart. It was his signature scripture. “Who by his power formed the mountains and being armed with strength …”

There, now they could all relax. His sermon had included a reference to mountain. He continued reading Psalm 65, but even his own mind was wandering.

As a self-centred, outspoken 25-year-old, he had climbed Mount Everest.

Unfortunately, he wasn’t the first person to do so, and even though the achievement was commendable compared to what most people did in their lifetimes, he wasn’t exactly overwhelmed with offers from cereal companies to feature his face on their boxes of wheatie squares.

But a local church had approached him.

Would he be interested in sharing his experience with their youth group? No scripture references necessary. The youth pastor would have a few things to say about God and mountains and then Chris could go ahead and speak about Everest.

For lack of anything better to do, he had accepted.

That had been ten years ago. Since then, he had worked his way up from becoming a part-time assistant youth pastor, to a full-time assistant youth pastor, to a full-time youth pastor, and now, to a full-time assistant pastor.

And it was only a matter of time before he was the pastor.

Everyone thought of it as Chris’s church. He always made sure to include a scriptural reference to mountains, an admonition to scale seemingly impossible problems with perseverance, and a few anecdotes about his own climb (granted, he often had to adjust them slightly to suit the message). In fact, out on the signboard of the community church was a mountain peak beside its name. That had been added when he had built up the youth program from an apathetic twenty teenagers to an enthusiastic two hundred, or so.

But it had become increasingly challenging to handle the youth group. They were easily inspired. They wanted to climb mountains. The greatest difficulty for Chris was to take it from the abstract to the concrete. Climb what mountain? He had never come up with a good answer. He had found it easier to talk to the adults. The adults, he found, didn’t want to climb any mountains. Metaphorically, the seemed more inclined to seek out a bypass, to find the path of least exertion.

“The kid want to mission trip,” said Danny, comming into Chris’s office and sitting down. “All the big churches do it and we’ve got the youth for it.”

Chris nodded, only half-listening. He was scrolling through his emails.

Danny, the pastor, was an energetic man in his mid-fifties. He made no secret of the fact that his heart was with missions. In fact, that was why Chris was so sure that he would be the pastor in short time. Danny would probably turn into a full-time missionary in the near future. Chris wouldn’t be surprised if he was just using the kids as an excuse for a trip to Bolivia, or some other mission field that he would lead them to.

“We’re thinking of Nepal,” said Danny.

“Nepal?” Now Danny had Chris’s attention. “It’s not exactly friendly to the gospel.”

“That’s why the kids want to do it,” said Danny. “A group of them approached me and said they think it’s what God wants them to do.”

In his head, Chris disagreed. They had heard him speak more than once about his experience climbing Everest. Everest was in Nepal. It wasn’t God. It was just a natural outcome of being part of a church that made a mountain its logo.

“Well,” said Chris, his eyes straying back to his computer screen; despite that most of the emails were spam. “All the best to them. I take it Randy will lead them.”

Randy was the youth pastor, an enthusiastic man in his late twenties, recently married.

Danny shook his head.

“Susan just found out she’s pregnant.” Susan was Randy’s new wife. “He won’t be going anywhere for a while. Besides, the kids want you to lead it.” Chris’s eyes widened. Danny grinned. “I think some of them are hoping you’ll lead them on a little side trip up Everest.”

“That would be easier than a mission trip,” said Chris. “I don’t know much about the place. I mean, I spent more time in base camp than I did in Kathmandu, but I remember a lot of temples and prayer wheels and things. And guys going around in colourful robes and there were these poles people put up with scarves on them. One of the guides told me the were prayers and the wind blew the requests up to heaven. Or something like that. To be honest, I didn’t really pay attention.”

“I’ve been doing a little reading on the topic, ever since the kids came to me,” said Danny. “From what I understand, the real difficulty is the Maoist army that runs the country. They came to power in 2008 and promised to establish a democratic republic. The country is predominantly Hindu, but the prayer wheels that you saw would have been part of the failth of the country’s Buddhist minority, about 10%, as I recall. In any case, the Christians there face opposition from all sides.”

“Sounds fun,” said Chris grimly. “You avoid all that if you just go to climb the mountains.”

Danny nodded.

“I haven’t told the kis this, but evangelism there is seriously frowned on. If the indigenous people do it, they could be fined or imprisoned. If foreigners do it, it’s less severe, eviction from the country. But I think if some of the parents go online and start looking into it, they’ll never let their kids out sof the country.”

Chris was relieved.

This mission trip wouldn’t get off the ground.

android develepment

##1. Introduce

##2. Download

2.1 Download Tools

Download android SDK from http://developer.android.com/sdk/index.html, ADT
Bundle is contain SDK and Eclipse, Linux 64-bit platform use
http://dl.google.com/android/adt/adt-bundle-linux-x86_64-20130522.zip, if just
want to download SDK only, download http://dl.google.com/android/android-sdk_r22.0.1-linux.tgz
for Linux 32 & 64-bit, other platform please referent the download page instruction.

2.2 Setting Environment

Install android develep plugin:

https://dl-ssl.google.com/android/eclipse/

##3. Workflow

If you want to run you application on really device, you should read this page
http://developer.android.com/tools/device.html.

##4. Create Your First Application

##5. Run sample

##6. Improve your skills

##7. Reference:

use sg_ses

##1.Introduction

The sg_ses utility enables a user “to manage and sense the state of the power
supplies, cooling devices, displays, indicators, individual drives, and other
non-SCSI elements installed in an enclosure”.
The SCSI Enclosure Services standards (most recent is SES-2 ANSI INCITS 448-2008
) and the latest draft (ses3r03.pdf at www.t10.org) describe the format that the
sg_ses utility expects to find in a SES device (“logical unit” or “process”)

##2.Command Tools

sg_map - displays mapping between linux sg and other SCSI devices

sg_ses - send controls and fetch status from a SCSI EnclosureServices (SES) device

sg = SCSI Generic
ses = SCSI Enclosure Service

##3.Use command
query mapping of expander and device name

[root@ ~]# sg_map
/dev/sg0  /dev/sda
/dev/sg1  /dev/sdb
/dev/sg2  /dev/sdc
/dev/sg3  /dev/sdd
/dev/sg4  /dev/sde
/dev/sg5  /dev/sdf
/dev/sg6  /dev/sdg
/dev/sg7
/dev/sg8  /dev/sdh
/dev/sg9  /dev/sdi
/dev/sg10  /dev/sdj
/dev/sg11  /dev/sdk
/dev/sg12  /dev/sdl
/dev/sg13  /dev/sdm
/dev/sg14

[root@ ~]# sg_map -i
/dev/sg0  /dev/sda  ATA       SanDisk SSD U100  10.5
/dev/sg1  /dev/sdb  ATA       Hitachi HUA72201  A3EA
/dev/sg2  /dev/sdc  ATA       Hitachi HUA72201  A39C
/dev/sg3  /dev/sdd  ATA       ST2000VX000-9YW1  CV13
/dev/sg4  /dev/sde  ATA       Hitachi HUA72201  A3EA
/dev/sg5  /dev/sdf  ATA       ST2000VX000-9YW1  CV13
/dev/sg6  /dev/sdg  ATA       ST2000VX000-9YW1  CV13
/dev/sg7  GOOXI     Bobcat            0d00
/dev/sg8  /dev/sdh  ATA       ST2000VX000-9YW1  CV13
/dev/sg9  /dev/sdi  ATA       ST2000VX000-9YW1  CV13
/dev/sg10  /dev/sdj  ATA       ST2000VX000-9YW1  CV13
/dev/sg11  /dev/sdk  ATA       ST2000VX000-9YW1  CV13
/dev/sg12  /dev/sdl  ATA       Hitachi HDE72101  A31B
/dev/sg13  /dev/sdm  ATA       Hitachi HDE72101  A3AA
/dev/sg14  GOOXI     Bobcat            0d00

There are two expanders, /dev/sg7 and /dev/sg14, to show the only expanders
list, use below command:

[root@ ~]# sg_map | awk '{if($2==""){print $1}}'
/dev/sg7
/dev/sg14

We also can use ls to find expanders:

[root@ ~]# ls /dev/sg* -l
crw-rw---- 1 root disk 21,  0 Jun 18 22:54 /dev/sg0
crw-rw---- 1 root disk 21,  1 Jun 19 17:33 /dev/sg1
crw-rw---- 1 root disk 21, 15 Jun 19 23:33 /dev/sg15
crw-rw---- 1 root disk 21, 16 Jun 19 23:33 /dev/sg16
crw-rw---- 1 root disk 21, 17 Jun 19 23:33 /dev/sg17
crw-rw---- 1 root disk 21, 18 Jun 19 23:33 /dev/sg18
crw-rw---- 1 root disk 21,  2 Jun 18 22:55 /dev/sg2
crw-rw---- 1 root disk 21,  3 Jun 18 22:55 /dev/sg3
crw-rw---- 1 root disk 21,  4 Jun 19 17:33 /dev/sg4
crw-rw---- 1 root disk 21,  5 Jun 18 22:55 /dev/sg5
crw-rw---- 1 root disk 21,  6 Jun 20 17:49 /dev/sg6
crw-rw---- 1 root root 21,  7 Jun 18 22:55 /dev/sg7
crw-rw---- 1 root disk 21,  8 Jun 20 21:46 /dev/sg8
crw-rw---- 1 root disk 21,  9 Jun 20 21:46 /dev/sg9

[root@ ~]# ls /dev/sg* -l |awk '{if($4=="root"){print $10}}'
/dev/sg7

To view the pages of /dev/sg7

[root@ ~]# sg_ses -p 0 /dev/sg7
  GOOXI     Bobcat            0d00
    enclosure services device
Supported diagnostic pages:
  Supported diagnostic pages [0x0]
  Configuration (SES) [0x1]
  Enclosure status/control (SES) [0x2]
  String In/Out (SES) [0x4]
  Threshold In/Out (SES) [0x5]
  Element descriptor (SES) [0x7]
  Additional element status (SES-2) [0xa]
  Supported SES diagnostic pages (SES-2) [0xd]
  Download microcode (SES-2) [0xe]
  Subenclosure nickname (SES-2) [0xf]

To view the enclosure status

[root@ ~]# sg_ses -p 2 /dev/sg7
  GOOXI     Bobcat            0d00
    enclosure services device
Enclosure status diagnostic page:
  INVOP=0, INFO=1, NON-CRIT=0, CRIT=0, UNRECOV=0
  generation code: 0x0
  status descriptor list
    Element type: Array device slot, subenclosure id: 0
     Overall 0 descriptor:
       Predicted failure=0, Disabled=0, Swap=0, status: Unsupported
       OK=0, Reserved device=0, Hot spare=0, Cons check=0
       In crit array=0, In failed array=0, Rebuild/remap=0, R/R abort=0
       App client bypass A=0, Do not remove=0, Enc bypass A=0, Enc bypass B=0
       Ready to insert=0, RMV=0, Ident=0, Report=0
       App client bypass B=0, Fault sensed=0, Fault reqstd=0, Device off=0
       Bypassed A=0, Bypassed B=0, Dev bypassed A=0, Dev bypassed B=0
     Element 0 descriptor:
       Predicted failure=0, Disabled=0, Swap=0, status: Not installed
       OK=0, Reserved device=0, Hot spare=0, Cons check=0
       In crit array=0, In failed array=0, Rebuild/remap=0, R/R abort=0
       App client bypass A=0, Do not remove=0, Enc bypass A=0, Enc bypass B=0
       Ready to insert=0, RMV=0, Ident=0, Report=0
       App client bypass B=0, Fault sensed=0, Fault reqstd=0, Device off=0
       Bypassed A=0, Bypassed B=0, Dev bypassed A=0, Dev bypassed B=0
     Element 1 descriptor:
       Predicted failure=0, Disabled=0, Swap=0, status: OK
       OK=0, Reserved device=0, Hot spare=0, Cons check=0
       In crit array=0, In failed array=0, Rebuild/remap=0, R/R abort=0
       App client bypass A=0, Do not remove=0, Enc bypass A=0, Enc bypass B=0
       Ready to insert=0, RMV=0, Ident=0, Report=0
       App client bypass B=0, Fault sensed=0, Fault reqstd=0, Device off=0
       Bypassed A=0, Bypassed B=0, Dev bypassed A=0, Dev bypassed B=0

     ...

     Element 27 descriptor:
       Predicted failure=0, Disabled=0, Swap=0, status: Not installed
       OK=0, Reserved device=0, Hot spare=0, Cons check=0
       In crit array=0, In failed array=0, Rebuild/remap=0, R/R abort=0
       App client bypass A=0, Do not remove=0, Enc bypass A=0, Enc bypass B=0
       Ready to insert=0, RMV=0, Ident=0, Report=0
       App client bypass B=0, Fault sensed=0, Fault reqstd=0, Device off=0
       Bypassed A=0, Bypassed B=0, Dev bypassed A=0, Dev bypassed B=0

To view the enclosure status by specify index

[root@ ~]# sg_ses -p 2 -I 27 /dev/sg7
  GOOXI     Bobcat            0d00
    enclosure services device
Enclosure status diagnostic page:
  INVOP=0, INFO=1, NON-CRIT=0, CRIT=0, UNRECOV=0
  generation code: 0x0
  status descriptor list
     Element 27 descriptor:
       Predicted failure=0, Disabled=0, Swap=0, status: Not installed
       OK=0, Reserved device=0, Hot spare=0, Cons check=0
       In crit array=0, In failed array=0, Rebuild/remap=0, R/R abort=0
       App client bypass A=0, Do not remove=0, Enc bypass A=0, Enc bypass B=0
       Ready to insert=0, RMV=0, Ident=0, Report=0
       App client bypass B=0, Fault sensed=0, Fault reqstd=0, Device off=0
       Bypassed A=0, Bypassed B=0, Dev bypassed A=0, Dev bypassed B=0

view enclosure Additional element status
From result info, we can map the slot num with element index, also SAS address

[root@ ~]# sg_ses -p 0xa /dev/sg7
  GOOXI     Bobcat            0d00
    enclosure services device
Additional element status diagnostic page:
  generation code: 0x0
  additional element status descriptor list
      Element index: 0
        Transport protocol: SAS
        number of phys: 1, not all phys: 0, device slot number: 21
        phy index: 0
          device type: no device attached
          initiator port for:
          target port for:
          attached SAS address: 0x0000000000000000
          SAS address: 0x0000000000000000
          phy identifier: 0x0
      Element index: 1
        Transport protocol: SAS
        number of phys: 1, not all phys: 0, device slot number: 20
        phy index: 0
          device type: no device attached
          initiator port for:
          target port for: SATA_device
          attached SAS address: 0x500605b0000272bf
          SAS address: 0x500605b0000272a1
          phy identifier: 0x0

      ...

      Element index: 27
        Transport protocol: SAS
        number of phys: 1, not all phys: 0, device slot number: 22
        phy index: 0
          device type: no device attached
          initiator port for:
          target port for:
          attached SAS address: 0x0000000000000000
          SAS address: 0x0000000000000000
          phy identifier: 0x0

# Also, we can show the specify index additional element status 
[root@ ~]# sg_ses -p 0xa -I 26 /dev/sg7
  GOOXI     Bobcat            0d00
    enclosure services device
Additional element status diagnostic page:
  generation code: 0x0
  additional element status descriptor list
      Element index: 26
        Transport protocol: SAS
        number of phys: 1, not all phys: 0, device slot number: 23
        phy index: 0
          device type: no device attached
          initiator port for:
          target port for: SATA_device
          attached SAS address: 0x500605b0000272bf
          SAS address: 0x500605b0000272ba
          phy identifier: 0x0

This command get a simple output:

# command explain:
# grep -E 'Element|slot'     search by an extended regular expression 
# sed 'N;s/\n//'             join two lines together
# awk '{print $3,$15}'       printf third and 15th column

# out format: element_index slot_number 
[root@ ~]# sg_ses -p 0xa /dev/sg7 |grep -E 'slot|Element' |sed 'N;s/\n//' |awk '{print $3,$15}'
0 21
1 20
2 16
3 12
4 24
5 25
6 26
7 27
8 8
9 4
10 0
11 1
12 3
13 2
14 7
15 6
16 5
17 11
18 10
19 9
20 15
21 14
22 13
23 19
24 18
25 17
26 23
27 22

# out format: slot_number element_index
[root@ ~]# sg_ses -p 0xa /dev/sg7 |grep -E 'slot|Element' |sed 'N;s/\n//' |awk '{print $15,$3}' |sort -n
0 10
1 11
2 13
3 12
4 9
5 16
6 15
7 14
8 8
9 19
10 18
11 17
12 3
13 22
14 21
15 20
16 2
17 25
18 24
19 23
20 1
21 0
22 27
23 26
24 4
25 5
26 6
27 7

we can get the mapping relation between slot number and SAS address:

# 0x0000000000000000 means there is not exist physical disk
[root@ ~]# sg_ses -p 0xa /dev/sg7 |grep -E 'slot|  SAS address' |sed 'N;s/\n//' |awk '{print $12,$15}' |sort -n
0 0x50014ee300165dde
1 0x50014ee3556bb24a
2 0x50014ee30016698a
3 0x50014ee300166666
4 0x50014ee300165dea
5 0x50014ee3aac1019e
6 0x0000000000000000
7 0x0000000000000000
8 0x0000000000000000
9 0x0000000000000000
10 0x0000000000000000
11 0x0000000000000000
12 0x0000000000000000
13 0x0000000000000000
14 0x0000000000000000
15 0x0000000000000000
16 0x500605b0000272a2
17 0x500605b0000272b9
18 0x500605b0000272b8
19 0x500605b0000272b7
20 0x500605b0000272a1
21 0x0000000000000000
22 0x0000000000000000
23 0x500605b0000272ba
24 0x500148500018fba0
25 0x500148500018fba0
26 0x0000000000000000
27 0x500148500018fba0

[root@ ~]# sg_ses -p 0xa /dev/sg7 |grep -E 'slot|  SAS address' |sed 'N;s/\n//' |awk '{print $12,$15}' |sort -n |grep -v 0x0000000000000000
0 0x50014ee300165dde
1 0x50014ee3556bb24a
2 0x50014ee30016698a
3 0x50014ee300166666
4 0x50014ee300165dea
5 0x50014ee3aac1019e
16 0x500605b0000272a2
17 0x500605b0000272b9
18 0x500605b0000272b8
19 0x500605b0000272b7
20 0x500605b0000272a1
23 0x500605b0000272ba
24 0x500148500018fba0
25 0x500148500018fba0
27 0x500148500018fba0

##4.Control LED

Table 72 – Array Device Slot control element
(Taken from http://sg.danny.cz/sg/sg_ses.html)

+-Byte\Bit-+---7----+---6----+---5----+---4----+---3----+---2----+---1----+---0----+
+     0    +                           COMMON CONTROL                              +
+----------+--------+--------+--------+--------+--------+--------+--------+--------+
+          + RQST   + RQST   + RQST   + RQST   + RQST   +RQST IN +RQST    + RQST   +
+     1    + OK     + RSVD   + HOT    + CONS   + IN CRIT+FAILED  +REBUILD/+ R/R    +
+          +        + DEVICE + SPARE  + CHECK  + ARRAY  +ARRAY   +REMAP   + ABORT  +
+----------+--------+--------+--------+--------+--------+--------+--------+--------+
+     2    + RQST   + DO NOT +Reserved+ RQST   + RQST   + RQST   + RQST   +Reserved+
+          + ACTIVE + REMOVE +        +MISSING + INSERT + REMOVE + IDENT  +        +
+----------+--------+--------+--------+--------+--------+--------+--------+--------+
+     3    +     Reserved    + RQST   + DEVICE + ENABLE + ENABLE +     Reserved    +
+          +                 + FAULT  +  OFF   + BYP A  + BYP B  +                 +
+----------+--------+--------+--------+--------+--------+--------+--------+--------+

Insert harddisk, Blue LED light on, it is controlled by hardware.

Read/write harddisk, Blue LED flash, it is controlled by hardware also.

--index   : Element index (not slot index)
--set     : set   a status of specify element
--clear   : clear a status of specify element
/dev/sg5  : expander position (There is problem, how to position which disk
            belong to which expander?)

##raid failed: (Blue LED light on, green LED flash)
sg_ses --index=27 --set=1:2:1 /dev/sg5

##rebuild: (Red LED flash)
sg_ses --index=27 --set=1:1:1 /dev/sg5

##miss:
sg_ses --index=27 --set=2:4:1 /dev/sg5

##indent: (Blue and Green LED flash)
sg_ses --index=27 --set=2:1:1 /dev/sg5

##disk fault: (Red LED light on)
sg_ses --index=27 --set=fault  /dev/sg5
or
sg_ses --index=27 --set=3:5:1  /dev/sg5

all the upon command can use --clear option to clean status.

The latter three invocations use a numerical description of the field whose 
format is <start_byte>:<start_bit>[:<number_of_bits>] . 
The <number_of_bits> defaults to 1 when it is not given.(try to understand 
it with table 72)

#clear status
sg_ses --index 27 --clear=1:1 /dev/sg5
sg_ses --index 27 --clear=1:2 /dev/sg5
sg_ses --index 27 --clear=3:5 /dev/sg5

Some problems:

  • How can I know what color the LED would be when the command done?
  • Is there exist one command that can clear all status? (I do not want to use the
    clear command again and again, it is too ugly. or it is not need to do these?)

##5.Other command tools

获取磁盘信息的命令工具:

磁盘所包含的信息

硬件信息:型号、物理类型、序列号、温度、转速、工作速率、磁盘LU、
         磁盘端口SAS地址、厂商、固件、LED灯状态等
位置信息:Enclosure、slot
逻辑信息:运行状态(在线、空闲热备盘、重构盘)、逻辑类型(成员盘、空闲热备盘、
         空闲盘)、所属RAID组、所属卷组
SMART信息


#sg_map
显示sg设备和sd设备的映射关系

#sginfo –l
获取sg设备的信息

#sg_ses –p 0xa /dev/sg18
获取磁盘槽位号、磁盘端口SAS地址

#ll  /dev/disk/by-path
获取磁盘端口SAS地址、磁盘盘符

#sg_inq  -p  0x83  /dev/sdq
磁盘LU、磁盘端口SAS地址

#udevinfo –a –p /block/sdq
获取磁盘的信息

#udevinfo –q all –n /dev/sdq
获取磁盘的详细信息

#scsi_id  -x –g –s /block/sdq 
获取磁盘详细信息

#smartctl  -H /dev/sdb
查看磁盘健康状态

#smartctl -a  /dev/sdb
#smartctl -A /dev/sdb
查看磁盘SMART信息和温度

#smartctl -x  /dev/sdb
显示磁盘所有信息,包括SAS地址等

output status in raw format:

[root@ ~]# sg_ses -p 0x2 /dev/sg7 -r
    00 00 00 00 00 00 00 00  05 04 00 00 01 04 00 00
    01 01 00 00 05 00 00 00  05 00 00 00 05 00 00 00
    05 00 00 00 05 00 00 00  05 00 00 00 05 00 00 00
    01 00 00 00 01 00 00 00  01 00 00 00 01 00 00 00
    05 00 00 00 05 00 00 00  05 00 00 00 05 00 00 00
    05 00 00 00 05 00 00 00  05 00 00 00 05 00 00 00
    05 00 00 00 11 00 00 00  11 00 00 00 11 00 00 00
    01 00 00 00 05 00 00 00

byte 1 to 8 is general description, element description if start from 8th byte.

From ses3r05.pdf Table 63 -- Element status code field (Page 70)
01    0k            Element is installed and no error conditions are known.
05    Not Installed Element is not installed in enclosure.
09~0F Reserved

So, the first 4 bytes start from 8th byte is 05 04 00 00, means not disk installed 
and its status is "IN FAILED ARRAY".(See ses3r05.pdf page 77 Table 73)

The second 4 bytes is 01 04 00 00, means that disk is ok and status is "IN FAILED ARRAY".

The third 4 bytes is 01 01 00 00, means that disk ok and "R/R ABORT".

The 20th 4bytes is 11 00 00 00, means disk ok. 

Notice that the frist byte 11 is equal to 01, look at Table 63, we can see 
that element status code only depends on low 4 bits (0~3th bits for ELEMENT 
STATUS CODE, 4th bit is SWAP, 5th is DISABLED, 6th is PRDFAIL, 7th is Reserved)

look ses3r05.pdf for more details:

Table 12 — Enclosure Status diagnostic page -- Page 30
Table 13 — Status descriptor                -- Page 32
Table 62 — Status element format            -- Page 70
Table 63 — ELEMENT STATUS CODE field        -- Page 70
Table 72 — Array Device Slot control element-- Page 76
Table 73 — Array Device Slot status element -- Page 77

##7.Reference

iSCSI and implementation

Introduce

iSCSI(发音为 /аɪskʌzi/)(iSCSI = internet Small Computer System Interface )
又稱為IP-SAN,是一种基于因特网及SCSI-3协议下的存储技术,
由IETF提出,並於2003年2月11日成為正式的標準。
与传统的SCSI技术比较起来,iSCSI技术有以下三个革命性的变化:

1. 把原來只用於本機的SCSI協同透過TCP/IP网络傳送,使连接距离可作无限的地域延伸
2. 连接的服务器数量无限(原來的SCSI-3的上限是15)
3. 由于是服务器架构,因此也可以实现在线扩容以至动态部署

iSCSI利用了TCP/IP的port 860 和 3260 作为沟通的渠道。透过两部计算机之间利用iSCSI
的协议来交换SCSI命令,让计算机可以透过高速的局域网集线来把SAN模拟成为本地的储存装置。

The overall structure of an iSCSI PDU is as follows:

Byte/     0       |       1       |       2       |       3       |
   /              |               |               |               |
  |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|
  +---------------+---------------+---------------+---------------+
 0/ Basic Header Segment (BHS)                                    /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
48/ Additional Header Segment 1 (AHS)  (optional)                 /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
  / Additional Header Segment 2 (AHS)  (optional)                 /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
 ----
  +---------------+---------------+---------------+---------------+
  / Additional Header Segment n (AHS)  (optional)                 /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
 ----
  +---------------+---------------+---------------+---------------+
 k/ Header-Digest (optional)                                      /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
 l/ Data Segment(optional)                                        /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
 m/ Data-Digest (optional)                                        /
 +/                                                               /
  +---------------+---------------+---------------+---------------+

The format of the BHS(fixed size, 48 bytes ) is:

Byte/     0       |       1       |       2       |       3       |
   /              |               |               |               |
  |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|
  +---------------+---------------+---------------+---------------+
 0|.|I| Opcode    |F|  Opcode-specific fields                     |
  +---------------+---------------+---------------+---------------+
 4|TotalAHSLength | DataSegmentLength                             |
  +---------------+---------------+---------------+---------------+
 8| LUN or Opcode-specific fields                                 |
  +                                                               +
12|                                                               |
  +---------------+---------------+---------------+---------------+
16| Initiator Task Tag                                            |
  +---------------+---------------+---------------+---------------+
20/ Opcode-specific fields                                        /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
48

For request PDUs, the I bit set to 1 is an immediate delivery marker.

Initiator opcodes defined in this specification are:

0x00 NOP-Out
0x01 SCSI Command (encapsulates a SCSI Command Descriptor Block)
0x02 SCSI Task Management function request
0x03 Login Request
0x04 Text Request
0x05 SCSI Data-Out (for WRITE operations)
0x06 Logout Request
0x10 SNACK Request
0x1c-0x1e Vendor specific codes

Target opcodes are:

0x20 NOP-In
0x21 SCSI Response - contains SCSI status and possibly sense
     information or other response information.
0x22 SCSI Task Management function response
0x23 Login Response
0x24 Text Response
0x25 SCSI Data-In - for READ operations.
0x26 Logout Response
0x31 Ready To Transfer (R2T) - sent by target when it is ready
     to receive data.
0x32 Asynchronous Message - sent by target to indicate certain
     special conditions.
0x3c-0x3e Vendor specific codes
0x3f Reject

When set to 1 it indicates the final (or only) PDU of a sequence.

Some opcodes operate on a specific Logical Unit. The Logical Unit
Number (LUN) field identifies which Logical Unit. If the opcode does
not relate to a Logical Unit, this field is either ignored or may be
used in an opcode specific way. The LUN field is 64-bits and should
be formatted in accordance with [SAM2]. For example, LUN[0] from
[SAM2] is BHS byte 8 and so on up to LUN[7] from [SAM2], which is BHS
byte 15.

The general format of an AHS is:

Byte/     0       |       1       |       2       |       3       |
   /              |               |               |               |
  |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|
  +---------------+---------------+---------------+---------------+
 0| AHSLength                     | AHSType       | AHS-Specific  |
  +---------------+---------------+---------------+---------------+
 4/ AHS-Specific                                                  /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
 x

The format of the SCSI Command PDU is:

Byte/     0       |       1       |       2       |       3       |
   /              |               |               |               |
  |0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|0 1 2 3 4 5 6 7|
  +---------------+---------------+---------------+---------------+
 0|.|I| 0x01      |F|R|W|. .|ATTR | Reserved                      |
  +---------------+---------------+---------------+---------------+
 4|TotalAHSLength | DataSegmentLength                             |
  +---------------+---------------+---------------+---------------+
 8| Logical Unit Number (LUN)                                     |
  +                                                               +
12|                                                               |
  +---------------+---------------+---------------+---------------+
16| Initiator Task Tag                                            |
  +---------------+---------------+---------------+---------------+
20| Expected Data Transfer Length                                 |
  +---------------+---------------+---------------+---------------+
24| CmdSN                                                         |
  +---------------+---------------+---------------+---------------+
28| ExpStatSN                                                     |
  +---------------+---------------+---------------+---------------+
32/ SCSI Command Descriptor Block (CDB)                           /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
48/ AHS (Optional)                                                /
  +---------------+---------------+---------------+---------------+
 x/ Header Digest (Optional)                                      /
  +---------------+---------------+---------------+---------------+
 y/ (DataSegment, Command Data) (Optional)                        /
 +/                                                               /
  +---------------+---------------+---------------+---------------+
 z/ Data Digest (Optional)                                        /
  +---------------+---------------+---------------+---------------+

Reference:

Linux Kernel Linked List Explained

Using the list:

#include <stdio.h>
#include <stdlib.h>

#include "list.h"

struct kool_list{
    int to;
    struct list_head list;
    int from;
    };

int main(int argc, char **argv){
    struct kool_list *tmp;
    struct list_head *pos, *q;
    unsigned int i;

    struct kool_list mylist;
    INIT_LIST_HEAD(&mylist.list);

    for(i=5; i!=0; --i){
        tmp=(struct kool_list*)malloc(sizeof(struct kool_list));
        printf("enter to and from:");
        scanf("%d %d", &tmp->to, &tmp->from);
        list_add(&(tmp->list), &(mylist.list));
    }
    printf("\n");

    printf("traversing the list using list_for_each_entry()\n");
    list_for_each_entry(tmp, &mylist.list, list)
        printf("to=%d from=%d\n", tmp->to, tmp->from);
    printf("\n");

    printf("deleting the list using list_for_each_safe()\n");
    list_for_each_safe(pos, q, &mylist.list){
        tmp=list_entry(pos, struct kool_list, list);
        printf("freeing item to=%d from=%d\n", tmp->to, tmp->from);
        list_del(pos);
        free(tmp);
    }

    return 0;
}

Testing, as below:

[dennis@localhost test]$ ./main
enter to and from:19 9
enter to and from:18 8
enter to and from:17 7
enter to and from:16 6
enter to and from:15 5

traversing the list using list_for_each_entry()
to=15 from=5
to=16 from=6
to=17 from=7
to=18 from=8
to=19 from=9

deleting the list using list_for_each_safe()
freeing item to=15 from=5
freeing item to=16 from=6
freeing item to=17 from=7
freeing item to=18 from=8
freeing item to=19 from=9
[dennis@localhost test]$

How it work? go to the reference website for detail.

Reference: