apache avro analyzed

analyzed apace avro c code

////////////////////////////////////////////////////////////
// Apache Avro c实现代码分析
// 
// By matrix207 2015-08-03 at home
// 
////////////////////////////////////////////////////////////

/////////////////////// Part 1 /////////////////////////////
文件                             功能
--------------------------------------------------------------------------
allocation.c allocation.h   一个提供申请、追加和释放内存功能的接口实现
                            参考Lua的解析器实现
array.c
avroappend.c                例子?
avrocat.c                   例子?
avro_generic_internal.h
avro.h                      include一些头文件
avromod.c                   应该是测试压缩一个schema文件的例子?
avropipe.c                  例子?
avro_private.h              一些宏,感觉没有特别用处
codec.h codec.c             编码器, 支持三种压缩算法:DEFLATE、LZMA和SNAPPY
consume-binary.c
consumer.h consumer.c       消费者???
datafile.c
datum.h datum.c             
datum_equal.c
datum_read.c
datum_size.c
datum_skip.c
datum_validate.c
datum_value.c
datum_write.c
dump.c
dump.h
encoding_binary.c
encoding.h
errors.h errors.c           错误信息,支持设置和提取功能,支持在已有的错误信息前插入新的错误信息.
generic.h generic.c         内容很多,还要继续研究????
io.h io.c                   io类型,包含文件io和内存io
map.c                       map数据操作的实现
memoize.c                   avro_memoize_t数据类型操作的实现
resolved-reader.c
resolved-writer.c
resolver.h resolver.c
schema.h schema.c
schema_equal.c
schema_specific.c
st.h st.c                   通用hash表实现
string.c                    使用wrapper buffer的方式创建一个类似string的数据avro_raw_string_t
                            这里主要实现相关操作
value.h value.c
value-hash.c                value的hash操作,功能是??
value-json.c                把二进制格式数据(datum)转化为支持UTF-8的JSON格式数据(avrocat中有使用)
value-read.c                定义方法avro_value_read, 供datafile.c使用,avro_file_writer_open用到
value-sizeof.c              定义方法avro_value_sizeof, 没有找到使用者???
value-write.c               定义方法avro_value_write,供datum_write.c和datafile.c使用
wrapped-buffer.c            buffer操作

msinttypes.h                windows平台文件
platform.h                  平台文件, 针对不同平台include不同的头文件
basics.h                    简单和复杂数据类型的枚举型定义、schema和datum类型定义、对象类型定义
                            各种宏来判断数据类型(简单和复杂类型)
data.h                      各种数据类型的结构体定义(array、Maps、Wrapped buffer、strings、Memoization
legacy.h                    为了兼容旧接口而保留的接口(新代码不会使用)
msstdint.h                  windows平台文件
refcount.h                  引用计数
schema.h                    一些函数声明而已

////////////////////////////////////////////////////////////
重点,理解这些代码, from encoding_binary.c

static int write_long(avro_writer_t writer, int64_t l)
{
    char buf[MAX_VARINT_BUF_SIZE];
    uint8_t bytes_written = 0;
    uint64_t n = (l << 1) ^ (l >> 63);
    while (n & ~0x7F) {
        buf[bytes_written++] = (char)((((uint8_t) n) & 0x7F) | 0x80);
        n >>= 7;
    }
    buf[bytes_written++] = (char)n;
    AVRO_WRITE(writer, buf, bytes_written);
    return 0;
}

core dumped

What is coredump

How to analyse coredump

Why program coredump

  • memory overload
  • using thread-unsafe functions on multi-thread
  • not using lock when read-write data in multi-thread
  • invalid point
  • stack overload

How to use core file

  • allow to generate core file
    • ulimit –c unlimited
    • echo /tmp/core.%e.%p > /proc/sys/kernel/core_pattern
    • gcc –g core_dump_test.c -o core_dump_test, must used -g to build.
  • get core file
    • ./test
  • debug
    • gdb ./test /tmp/core
    • bt
    • f 3
    • p buffer

Signal of core

  • man 7 signal

Examples

  • write data to protected memory

    #include <stdio.h>
    int main(){
       int i=0;
       scanf("%d",i);
       printf("%d\n",i);
       return 0;
    }
    
    #include <stdio.h>
    int main(){
       char *p;
       p = NULL;
       *p = 'x';
       printf("%c", *p);
       return 0;
    }
    
  • memory overflow (array overflow, variable type different and so on)

    #include <stdio.h>
    int main(){
        char test[1];
        printf("%c", test[1000000000]);
        return 0;
    }
    

Access to a not exist address

#include <stdio.h>
 int main(){
      int b = 10;
      printf("%s\n", b);
      return 0;
}

#include <stdio.h>
#include <string.h>
  int main(){
     char c='c';
     int i=10;
     char buf[100];
     printf("%s", c); //试图把char型按照字符串格式输出,这里的字符会解释成整数,
                           //再解释成地址,所以原因同上面那个例子
     printf("%s", i); //试图把int型按照字符串输出
     memset(buf, 0, 100);
     sprintf(buf, "%s", c);  试图把char型按照字符串格式转换
     memset(buf, 0, 100);
     sprintf(buf, "%s", i);//试图把int型按照字符串转换
}
  • Other, likes:
    • Initialization when defined point, check point before using.
    • Careful array index
    • check if success when do pthread_create, if fail and do pthread_join for
      this thread will occur segment fault

Auto start gdb when core occur

#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <string.h>
#include <sys/types.h>
#include <unistd.h>

void dump(int signo)
{
    char buf[1024];
    char cmd[1024];
    FILE *fh;

    snprintf(buf, sizeof(buf), "/proc/%d/cmdline", getpid());
    if(!(fh = fopen(buf, "r")))
        exit(0);
    if(!fgets(buf, sizeof(buf), fh))
        exit(0);
    fclose(fh);
    if(buf[strlen(buf) - 1] == '\n')
        buf[strlen(buf) - 1] = '\0';
    snprintf(cmd, sizeof(cmd), "gdb %s %d", buf, getpid());
    system(cmd);

    exit(0);
}

void dummy_function (void)
{
    unsigned char *ptr = 0x00;
    *ptr = 0x00;
}

int main (void)
{
    signal(SIGSEGV, &dump);
    dummy_function ();

    return 0;
}

Reference

difference between stack and heap

Difference

  • stack
    • allocate memory on stack by operate system
    • Have limit size, and less than 1MB always
  • heap
    • allocate memeory on heap
    • using malloc or new to allocate memory
    • user and defined the size, and the limit depend on virtual memory size
    • prone to memory fragments

What is memory fragments

  • Internal fragmentation
    • memory allocation provided memory to program in chunks divisible by 4, 8 or 16
    • if application request 23 bytes, it actually get a chunk of 32 bytes.
    • Really? How to prove it? And what happen if I using memory not belong to me?
  • External fragmentation
    • cause by free called
    • An example, first, you allocate 9 bytes memory, then allocate another 5 bytes,
      and later you free the first 9 bytes, but now, you want to allocate 10 bytes,
      this time the first 9 bytes memory can’t be used, because it is not enough.
      Allocation must using continues memory, so it would allocate another 10 bytes
      behind the 5 bytes position. Now the first block memory is memory fragments.

How to avoid memory fragments

Why memory overflow but not error occur?

  • sample code

    #include <stdio.h>
    #include <stdlib.h>
    #include <string.h>
    
    void main()
    {
        // allocate 13 bytes memory on heap
        char* p = (char*)malloc(13);
        char* c_12 = "012345678912";
        char* c_15 = "012345678912345";
        char* c_19 = "0123456789123456789";
        char* c_99 = "aaaaaaaaaaaaaaaaaaaa"
        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
        "aaaaaaaaaaaaa";
    
        printf("do copy 12 bytes to p\n");
        strncpy(p, c_12, strlen(c_12)+1);
        p[12] = '\0'; printf("p=%s\n", p);
    
        printf("do copy 15 bytes to p\n");
        strncpy(p, c_15, strlen(c_15)+1);
        p[16] = '\0'; printf("p=%s\n", p);
    
        printf("do copy 19 bytes to p\n");
        strncpy(p, c_19, strlen(c_19)+1);
        p[20] = '\0'; printf("p=%s\n", p);
    
        printf("do copy 99 bytes to p\n");
        strncpy(p, c_99, strlen(c_99)+1);
        p[100] = '\0'; printf("p=%s\n", p);
    
        printf("Hello world\n");
    }
    

Reference

schema data model

Data model

Why need it

Schema Node

  • Define
    • R : root node (record node)
    • S : single node (child node, subschema)
    • A : array node (child node, subschema)
  • Example
    • R, only root node
    • R -> S1, only a singal node under root
    • R -> A1, only an array node under root
    • R -> A1 -> A2
    • R -> A1 -> S1
    • R -> S1 -> A1
    • R -> S1 -> S2
    • R -> S1 -> A1 -> S2
    • R -> A1 -> S1 -> A2
  • How to find the last node info by index of node?
    • step 0, set root node as parent node
    • step 1, use parent node to find array index
    • step 2, found array node, set this node to new parent node, go to step 1
    • step 3, use the parent node to find the last node
    • step 4, do some operations for the node
      • set value to fields and insert. (Using for add a new object)
    • Notices
      • only root node
  • How to create a key
  • How to write object
  • How to read object
  • How to update object
  • What is difference between full and delta object
  • Since there are read and write interfaces, why not insert and delete

Reference

How source code turn to be an executable program

What is an executable program

  • On linux, the format of executable program is ELF (Executable and Linking Format)
  • members of executable program
    • text segment
    • data segment
    • bss segment

How source code be compiled

Relation of source code, object file, library file, bin file

How bin file to load library

  • dynamic library
  • static library

Example

  • Hello world, a.c

    #include <stdio.h>
    int main()
    {
        printf ("Hello world\n");
    }
    
  • compile

    [dennis@localhost ~]$ gcc a.c
    [dennis@localhost ~]$ objdump -f a.out 
    
    a.out:     file format elf64-x86-64
    architecture: i386:x86-64, flags 0x00000112:
    EXEC_P, HAS_SYMS, D_PAGED
    start address 0x0000000000400440
    [dennis@localhost ~]$ file a.out
    a.out: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 2.6.32, BuildID[sha1]=7396e4d6e7100d1536548d05ba91a67255bf9592, not stripped
    [dennis@localhost ~]$ size a.out
       text       data        bss        dec        hex    filename
       1226        548          4       1778        6f2    a.out
    [dennis@localhost ~]$ readelf -h a.out
    ELF Header:
      Magic:   7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 
      Class:                             ELF64
      Data:                              2's complement, little endian
      Version:                           1 (current)
      OS/ABI:                            UNIX - System V
      ABI Version:                       0
      Type:                              EXEC (Executable file)
      Machine:                           Advanced Micro Devices X86-64
      Version:                           0x1
      Entry point address:               0x400440
      Start of program headers:          64 (bytes into file)
      Start of section headers:          4504 (bytes into file)
      Flags:                             0x0
      Size of this header:               64 (bytes)
      Size of program headers:           56 (bytes)
      Number of program headers:         9
      Size of section headers:           64 (bytes)
      Number of section headers:         30
      Section header string table index: 27
    [dennis@localhost ~]$ readelf -S a.out
    

Reference

nfs

About NFS

NFS is “Network File System”.

How to use

  • On server:

    • service nfs start|stop
    • set ACL (Access Control Lists) of files and directories.
      • setfacl -m other::— PATH
      • setfacl -m g:GROUP_NAME:rwx PATH
      • setfacl -m g:GROUP_NAME:— PATH
  • On clinet: mount nfs share disk

    • showmount -e 172.16.130.143
    • mkdir /mnt/nfs
    • mount 172.16.130.143:/share/vdshare001 /mnt/nfs
    • ls /mnt/nfs
    • umount /mnt/nfs
  • command operation:

    [root@Ustor ~]# service nfs start
    Starting RPC svcgssd: [ OK ]
    Starting NFS services: [ OK ]
    Starting NFS quotas: [ OK ]
    Starting NFS mountd: [ OK ]
    Stopping RPC idmapd: [ OK ]
    Starting RPC idmapd: [ OK ]
    Starting NFS daemon: [ OK ]
    [root@Ustor ~]# service nfs status
    rpc.svcgssd (pid 11647) is running…
    rpc.mountd (pid 11657) is running…
    nfsd (pid 11722 11721 11720 11719 11718 11717 11716 11715) is running…
    rpc.rquotad (pid 11653) is running..
    [root@Ustor ~]# ps aux |grep nfs
    root 11713 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd4]
    root 11714 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd4_callbacks]
    root 11715 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11716 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11717 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11718 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11719 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11720 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11721 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11722 0.0 0.0 0 0 ? S 23:34 0:00 [nfsd]
    root 11904 0.0 0.0 103240 844 pts/1 S+ 23:35 0:00 grep nfs
    [root@Ustor ~]# netstat -apn |grep 2049
    tcp 0 0 0.0.0.0:2049 0.0.0.0: LISTEN -
    tcp 0 0 :::2049 :::
    LISTEN -
    udp 0 0 0.0.0.0:2049 0.0.0.0: -
    udp 0 0 :::2049 :::
    -
    [root@Ustor ~]# service nfs stop
    Shutting down NFS daemon: [ OK ]
    Shutting down NFS mountd: [ OK ]
    Shutting down NFS quotas: [ OK ]
    Shutting down RPC svcgssd: [ OK ]

  • check status from /proc/fs/nfs and /proc/fs/nfsd/

Optimal

  • ★★★ nfs的传输速度优化

    • 设置块大小
      • time dd if=/dev/zero of=/testfs/testfile bs=8k count=1024 测试nfs写
      • time dd if=/testfs/testfile of=/dev/null bs=8k 测试nfs读
    • 网络传输包的大小
      • ping -s 2048 -f hostname, try different package size
      • nfsstat -o net
      • tracepath node1/端口号
      • ifconfig eth0
      • ifconfig eth0 mtu 16436, modify MTU value
      • /proc/sys/net/ipv4/ipfrag_high_thresh和/proc/sys/net/ipv4/ipfrag_low_thresh
    • nfs挂载的优化
      • timeo:如果超时,客户端等待的时间,以十分之一秒计算。
      • retrans:超时尝试的次数。
      • bg:后台挂载,很有用
      • hard:如果server端没有响应,那么客户端一直尝试挂载。
      • wsize:写块大小
      • rsize:读块大小
      • intr:可以中断不成功的挂载
      • noatime:不更新文件的inode访问时间,可以提高速度。
      • async:异步读写。
    • nfsd的个数
      • ps -efl |grep nfsd
      • vi /etc/init.d/nfs, modify RPCNFSDCOUNT
      • service nfs restart
    • nfsd的队列长度
      • /proc/sys/net/core/rmem_default
      • /proc/sys/net/core/rmem_max
      • /proc/sys/net/core/wmem_default
      • /proc/sys/net/core/wmem_max
      • vi /etc/sysctl.conf
    • mount 192.168.1.220:/mnt/nfs /mnt/nfs_t -o nolock, rsize=1024,wsize=1024,timeo=15
  • Linux NFS服务性能优化

  • NFS共享在使用前需要挂载(mount)。挂载时使用的参数很大程度上影响了读写的性能

      1. TCP or UDP
      1. rsize和wsize,每次读写的最大值
      1. sync和async. sync参数会强制wsize变成4KB,这会大大降低写性能。
  • 关于SAMBA和NFS是否支持断点续传

    • 经过测试,得出如下结论:
      • 1.samba 不支持
      • 2.nfs 支持
    • 测试环境和步骤如下:
    • 服务端:
      • 1.172.16.130.200, Centos7.0
    • Windows 客户端:
      • 1.yujl本地机器
      • 2.开始->运行->\172.16.130.200
      • 3.拷贝一个大的iso文件到xuni文件夹
      • 4.去机房拔掉172.16.130.200机器的网线
      • 5.发现本地的拷贝窗口停止传送数据
      • 6.接上网线,拷贝窗口仍然没有继续传送的迹象
    • Linux 客户端:
      • 1.我本地机器
      • 2.挂载172.16.130.200的/share/sambatest目录到本地的/mnt/nfs
      • 3.cp ~/Downloads/iso/CentOS-6.5-x86_64-bin-DVD1.iso /mnt/nfs/
      • 4.去机房拔掉172.16.130.200机器的网线
      • 5.确认本地无法连接172.16.130.200
      • 6.接上网线,使用ls -l /mnt/nfs/发现iso文件的大小还继续增加
      • 7.待cp完成后,md5sume计算两个iso文件,发现md5值一
    • 不支持,http://bbs.chinaunix.net/thread-4057603-1-1.html
    • http://bbs.chinaunix.net/thread-4057603-1-1.html

Code analysed(kernel version: linux-2.6.32-279.el6)

  • NFS 文件系统主要分为三个部分:The Protocol(网络协议),Client Side(NFS 客户端)和 Server Side(NFS 服务器)

    • NFS 客户端提供了接口,保证用户或者应用程序能像访问本地文件系统一样访问NFS 文件系统,
    • NFS 服务器作为数据源,为 NFS 客户端提供真实的文件系统服务
    • NFS 网络协议则使得 NFS 客户端和 NFS 服务器能够高效和可靠地进行通信
    • NFS 网络协议使用的是 RPC(Remote Procedure Call,远程过程调用)/XDR(External Data Representation,外部数据表示)机制
  • code file list:

    [dennis@localhost fs]$ pwd
    /home/dennis/work/kernel/linux-2.6.32-279.el6/fs
    [dennis@localhost fs]$ ls nfs
    cache_lib.c file.c nfs2xdr.c nfsroot.c
    cache_lib.h fscache.c nfs3acl.c pagelist.c
    callback.c fscache.h nfs3proc.c pnfs.c
    callback.h fscache-index.c nfs3xdr.c pnfs_dev.c
    callback_proc.c getroot.c nfs4filelayout.c pnfs.h
    callback_xdr.c idmap.c nfs4filelayoutdev.c proc.c
    client.c inode.c nfs4filelayout.h read.c
    delegation.c internal.h nfs4_fs.h super.c
    delegation.h iostat.h nfs4namespace.c symlink.c
    dir.c Kconfig nfs4proc.c sysctl.c
    direct.c Makefile nfs4renewd.c unlink.c
    dns_resolve.c mount_clnt.c nfs4state.c write.c
    dns_resolve.h namespace.c nfs4xdr.c
    [dennis@localhost fs]$ ls nfsd
    auth.c Makefile nfs4callback.c nfscache.c nfssvc.c xdr3.h
    auth.h nfs2acl.c nfs4idmap.c nfsctl.c nfsxdr.c xdr4.h
    cache.h nfs3acl.c nfs4proc.c nfsd.h state.h xdr.h
    export.c nfs3proc.c nfs4recover.c nfsfh.c stats.c
    Kconfig nfs3xdr.c nfs4state.c nfsfh.h vfs.c
    lockd.c nfs4acl.c nfs4xdr.c nfsproc.c vfs.h
    [dennis@localhost fs]$ ls nfs_common
    Makefile nfsacl.c
    [dennis@localhost fs]$ cd ../include/linux/
    [dennis@localhost linux]$ ls nfs*
    nfs2.h nfs4.h nfsd_idmap.h nfs_fs_sb.h nfs_iostat.h nfs_xdr.h
    nfs3.h nfs4_mount.h nfs_fs.h nfs.h nfs_mount.h
    nfs4_acl.h nfsacl.h nfs_fs_i.h nfs_idmap.h nfs_page.h

    nfsd:
    const.h debug.h export.h Kbuild nfsfh.h stats.h syscall.h

  • Client Side 源代码

    Client Side 的头文件在 include/linux/ 下面,C 文件在 fs/nfs 下面。

    dir.c/file.c/inode.c/symlink.c/unlink.c:与文件操作相关的系统调用
    read.c/write.c/flushd.c:文件读写
    mount_clnt.c/nfs_root.c:将 NFS 文件系统作为 root 目录的相关实现
    proc.c/nfs2xdr.c/nfs3proc.c/nfs3xdr.c:网络数据交换

与文件操作相关的系统调用都在 struct file_operations,struct inode_operations
这两个数据结构里面定义。文件的读操作 nfs_file_read 和写操作 nfs_file_write 被单
独提出来,因为文件读写性能将直接关系到文件系统的成败

  • Server Side 源代码

Server Side 的头文件在 include/linux/nfsd 下面,C 文件在 fs/nfsd 下面。

auth.c/lockd.c/export.c/nfsctl.c/nfscache.c/nfsfh.c/stats.c:导出目录的访问管理
nfssvc.c:NFS 服务 deamon 的实现
vfs.c:将 NFS 文件系统的操作转换成具体文件系统的操作
nfsproc.c/nfsxdr.c/nfs3proc.c/nfs3xdr.c:网络数据交换

导出目录的访问管理主要解决网络文件系统实现面临的几个重要问题,包括目录导出服务,
外部访问的权限控制,多客户端以及客户端与服务器的文件并发操作等。

  • process number of nfsd:

  • rename call stack:

    client: nfs_rename ->
    server: nfsd4_rename -> nfsd_rename

  • To be continue…

Reference

how to detect the internal network topology of company

What I want

How can know how many route/hub/switch are in my LAN network?

What I know

  • Host LAN IP is : 172.16.50.39
  • gateway is : 172.16.50.1
  • WLan IP: 219.134.89.155

Operations

[dennis@localhost ~]$ ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:23:ae:96:d6:9a brd ff:ff:ff:ff:ff:ff
    inet 172.16.50.39/24 brd 172.16.50.255 scope global enp2s0
       valid_lft forever preferred_lft forever
    inet6 fe80::223:aeff:fe96:d69a/64 scope link 
       valid_lft forever preferred_lft forever
[dennis@localhost ~]$ route
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
^C
[dennis@localhost ~]$ route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         172.16.50.1     0.0.0.0         UG    1024   0        0 enp2s0
172.16.50.0     0.0.0.0         255.255.255.0   U     0      0        0 enp2s0
[dennis@localhost ~]$ curl ifconfig.me
219.134.89.155
[dennis@localhost ~]$ ping www.baidu.com
PING www.a.shifen.com (180.97.33.108) 56(84) bytes of data.
64 bytes from 180.97.33.108: icmp_seq=1 ttl=51 time=36.6 ms
64 bytes from 180.97.33.108: icmp_seq=2 ttl=51 time=42.0 ms
...
^C
--- www.a.shifen.com ping statistics ---
120 packets transmitted, 120 received, 0% packet loss, time 138219ms
rtt min/avg/max/mdev = 28.061/41.513/82.900/10.308 ms
[dennis@localhost ~]$ traceroute www.baidu.com
traceroute to www.baidu.com (180.97.33.107), 30 hops max, 60 byte packets
 1  172.16.50.1 (172.16.50.1)  3.386 ms  3.431 ms  3.463 ms
 2  219.134.89.153 (219.134.89.153)  6.355 ms  7.169 ms  7.155 ms
 3  10.1.100.37 (10.1.100.37)  5.989 ms  6.274 ms  6.318 ms
 4  121.15.130.18 (121.15.130.18)  7.180 ms  7.167 ms  7.153 ms
 5  61.104.38.59.broad.fs.gd.dynamic.163data.com.cn (59.38.104.61)  6.227 ms  6.219 ms 57.104.38.59.broad.fs.gd.dynamic.163data.com.cn (59.38.104.57)  6.924 ms
 6  183.56.65.30 (183.56.65.30)  31.614 ms 183.56.66.2 (183.56.66.2)  13.175 ms 183.56.65.38 (183.56.65.38)  12.152 ms
 7  202.97.64.73 (202.97.64.73)  37.557 ms  37.543 ms  38.302 ms
 8  202.102.69.114 (202.102.69.114)  42.036 ms  43.259 ms  43.246 ms
 9  * * *
10  180.97.32.2 (180.97.32.2)  37.346 ms 180.97.32.6 (180.97.32.6)  38.140 ms 180.97.32.22 (180.97.32.22)  36.722 ms
11  * * *
12  * * *
13  * * *
14  * * *
15  * * *
16  * * *
17  * * *
18  * * *
19  * * *
20  * * *
21  * * *
22  * * *
23  * * *
24  * * *
25  * * *
26  * * *
27  * * *
28  * * *
29  * * *
30  * * *
[dennis@localhost ~]$ traceroute 219.134.89.155
traceroute to 219.134.89.155 (219.134.89.155), 30 hops max, 60 byte packets
 1  172.16.50.1 (172.16.50.1)  3.835 ms  3.856 ms  3.885 ms
 2  * * *
 3  * * *
 4  * * *
 5  * * *
 6  * * *
 7  * * *
 8  * * *
 9  * * *
10  * * *
11  * * *
12  * * *
13  * * *
14  * * *
15  * * *
16  * * *
17  * * *
18  * * *
19  * * *
20  * * *
21  * * *
22  * * *
23  * * *
24  * * *
25  * * *
26  * * *
27  * * *
28  * * *
29  * * *
30  * * *