Linux Netfilter and Traffic Control

Netfilter places

enum nf_inet_hooks {
        NF_INET_PRE_ROUTING,
        NF_INET_LOCAL_IN,
        NF_INET_FORWARD,
        NF_INET_LOCAL_OUT,
        NF_INET_POST_ROUTING,
        NF_INET_NUMHOOKS
};

Register the hooks

int nf_register_hook(struct nf_hook_ops *reg);
void nf_unregister_hook(struct nf_hook_ops *reg);
int nf_register_hooks(struct nf_hook_ops *reg, unsigned int n);
void nf_unregister_hooks(struct nf_hook_ops *reg, unsigned int n);

struct nf_hook_ops
{
        struct list_head list;

        /* User fills in from here down. */
        nf_hookfn *hook;
        struct module *owner;
        u_int8_t pf;
        unsigned int hooknum;
        /* Hooks are ordered in ascending priority. */
        int priority;
};

hook functions

typedef unsigned int nf_hookfn(unsigned int hooknum,
                               struct sk_buff *skb,
                               const struct net_device *in,
                               const struct net_device *out,
                               int (*okfn)(struct sk_buff *));

// <linux/netfilter.h>
#define NF_DROP 0
#define NF_ACCEPT 1
#define NF_STOLEN 2
#define NF_QUEUE 3
#define NF_REPEAT 4
#define NF_STOP 5
#define NF_MAX_VERDICT NF_STOP

pf

enum {
        NFPROTO_UNSPEC =  0,
        NFPROTO_IPV4   =  2,
        NFPROTO_ARP    =  3,
        NFPROTO_BRIDGE =  7,
        NFPROTO_IPV6   = 10,
        NFPROTO_DECNET = 12,
        NFPROTO_NUMPROTO,
};

hooknum

/* IP Hooks */
/* After promisc drops, checksum checks. */
#define NF_IP_PRE_ROUTING       0
/* If the packet is destined for this box. */
#define NF_IP_LOCAL_IN          1
/* If the packet is destined for another interface. */
#define NF_IP_FORWARD           2
/* Packets coming from a local process. */
#define NF_IP_LOCAL_OUT         3
/* Packets about to hit the wire. */
#define NF_IP_POST_ROUTING      4
#define NF_IP_NUMHOOKS          5

priority

enum nf_ip_hook_priorities {
        NF_IP_PRI_FIRST = INT_MIN,
        NF_IP_PRI_CONNTRACK_DEFRAG = -400,
        NF_IP_PRI_RAW = -300,
        NF_IP_PRI_SELINUX_FIRST = -225,
        NF_IP_PRI_CONNTRACK = -200,
        NF_IP_PRI_MANGLE = -150,
        NF_IP_PRI_NAT_DST = -100,
        NF_IP_PRI_FILTER = 0,
        NF_IP_PRI_SECURITY = 50,
        NF_IP_PRI_NAT_SRC = 100,
        NF_IP_PRI_SELINUX_LAST = 225,
        NF_IP_PRI_CONNTRACK_CONFIRM = INT_MAX,
        NF_IP_PRI_LAST = INT_MAX,
};

enum {
        NFPROTO_UNSPEC =  0,
        NFPROTO_IPV4   =  2,
        NFPROTO_ARP    =  3,
        NFPROTO_BRIDGE =  7,
        NFPROTO_IPV6   = 10,
        NFPROTO_DECNET = 12,
        NFPROTO_NUMPROTO,
};

Terminology

tc bandwidth rules

mbps = 1024 kbps = 1024 * 1024 bps => byte/s
mbit = 1024 kbit => kilo bit/s.
mb = 1024 kb = 1024 * 1024 b => byte
mbit = 1024 kbit => kilo bit.

Simple, classless Queueing Disciplines

• pfifo_fast
  

  这个队列的特点就象它的名字——先进先出(FIFO),也就是说没有任何数据包被特殊对待。它的参数与使用

• Token Bucket Filter
  

  令牌桶过滤器(TBF)是一个简单的队列规定:只允许以不超过事先设定的速率到来的数据包通过,但可能允许短暂突发流量朝过设定值。

  TBF 很精确,对于网络和处理器的影响都很小。所以如果您想对一个网卡限速, 它应该成为您的第一选择!

  TBF 的实现在于一个缓冲器(桶),不断地被一些叫做“令牌”的虚拟数据以特定速率(token rate)填充着。桶最重要的参数就是它的大小,也就是它能够存储令牌的数量。它的参数与使用.

• Stochastic Fairness Queueing
  

  SFQ(Stochastic Fairness Queueing,随机公平队列)是公平队列算法家族中的一个简单实现。它的精确性不如其它的方法,但是它在实现高度公平的同时,需要的计算量却很少。

  SFQ 的关键词是“会话”(或称作“流”) ,主要针对一个 TCP 会话或者 UDP 流。流量被分成相当多数量的 FIFO 队列中,每个队列对应一个会话。数据按照简单轮转的方式发送, 每个会话都按顺序得到发送机会。

  这种方式非常公平,保证了每一个会话都不会没其它会话所淹没。SFQ 之所以被称为“随机”,是因为它并不是真的为每一个会话创建一个队列,而是使用一个散列算法,把所有的会话映射到有限的几个队列中去。

  它的参数与使用.

Classful Queueing Disciplines

• How filters are used to classify traffic
  

            1:   root qdisc
             |
            1:1    child class
          /  |  \
         /   |   \
        /    |    \
        /    |    \
     1:10  1:11  1:12   child classes
      |      |     | 
      |     11:    |    leaf class
      |            | 
      10:         12:   qdisc
     /   \       /   \
  10:1  10:2   12:1  12:2   leaf classes
  

• Sample configuration
  

  创建这个树:

            1:   root qdisc
           / | \ 
         /   |   \
         /   |   \
       1:1  1:2  1:3    classes
        |    |    |
       10:  20:  30:    qdiscs    qdiscs
       sfq  tbf  sfq
  band  0    1    2
  

  # tc qdisc add dev eth0 root handle 1: prio
  ## 这个命令立即创建了类: 1:1, 1:2, 1:3
  # tc qdisc add dev eth0 parent 1:1 handle 10: sfq
  # tc qdisc add dev eth0 parent 1:2 handle 20: tbf rate 20kbit buffer 1600 limit 3000
  # tc qdisc add dev eth0 parent 1:3 handle 30: sfq
  

The Intermediate queueing device (IMQ)

• imq device
  

  /net/imq.ko
  

• netfilter support IMQ
  

  net/ipv4/netfilter/ipt_IMQ.ko
  net/ipv6/netfilter/ip6t_IMQ.ko
  

• iptables support IMQ
  

  iptables/libip6t_IMQ.so
  iptables/libipt_IMQ.so
  

• Sample configuration
  

  # modprobe imq numdevs=2
  # modprobe ipt_IMQ
  # ifconfig imq0 up
  # ifconfig imq1 up
  

  #!/bin/bash
  modprobe imq numdevs=2
  modprobe ipt_IMQ
  ifconfig imq0 up
  ifconfig imq1 up
  
  tc qdisc del dev imq0 root 2>/dev/null 1>&2
  tc qdisc del dev imq1 root 2>/dev/null 1>&2
  
  #IMQ 0
  tc qdisc add dev imq0 root handle 1: htb default 20
  
  tc class add dev imq0 parent 1: classid 1:1 htb rate 2mbit burst 15k
  
  tc class add dev imq0 parent 1:1 classid 1:10 htb rate 1mbit
  tc class add dev imq0 parent 1:1 classid 1:20 htb rate 1mbit
  
  tc qdisc add dev imq0 parent 1:10 handle 10: pfifo
  tc qdisc add dev imq0 parent 1:20 handle 20: sfq
  
  tc filter add dev imq0 parent 1:0 protocol ip prio 1 u32 match ip dst 192.168.228.30 flowid 1:10
  
  iptables -t mangle -A PREROUTING -i eth0 -j IMQ --todev 0
  
  #IMQ 1
  tc qdisc add dev imq1 root handle 2: htb default 20
  
  tc class add dev imq1 parent 2: classid 2:1 htb rate 10mbit burst 15k
  
  tc class add dev imq1 parent 2:1 classid 2:10 htb rate 1mbit ceil 10mbit
  tc class add dev imq1 parent 2:1 classid 2:20 htb rate 1mbit ceil 10mbit
  
  tc qdisc add dev imq1 parent 2:10 handle 10: pfifo
  tc qdisc add dev imq1 parent 2:20 handle 20: sfq
  
  tc filter add dev imq1 parent 2:0 protocol ip prio 1 u32 match ip dst 192.168.228.30 flowid 2:10
  
  iptables -t mangle -A POSTROUTING -o eth0 -j IMQ --todev 1
  

Advanced filters for (re-)classifying packets

• The u32 classifier
  

  U32 分类器是当前实现中最先进的过滤器。全部基于哈希表实现,所以当有很多过滤器的时候仍然能够保持健壮。

  用来配置过滤器的 tc 命令行由三部分组成:过滤器说明、选择器和动作。一个过滤器可以如下定义:

  tc filter add dev IF [ protocol PROTO ]
                       [ (preference|priority) PRIO ]
                       [ parent CBQ ]
  

  • U32 selector
    

    u32 选择器包含了能够对当前通过的数据包进行匹配的特征定义。它其实只是定义了 IP 包头中某些位的匹配而已,但这种看似简单的方法却非常有效。

    # tc filter add dev eth0 protocol ip parent 1:0 pref 10 u32 \
      match u32 00100000 00ff0000 at 0 flowid 1:10
    

  • General selectors
    

    一般选择器的语法是:

    match [ u32 | u16 | u8 ] PATTERN MASK at [OFFSET | nexthdr+OFFSET]
    

Examples

• 出口流限制
  

  ifconfig ath10 txqueuelen 32 up
  tc qdisc add dev ath10 handle 1: root htb
  tc class add dev ath10 parent 1: classid 1:1 htb rate 1000kbit mtu 10000
  tc filter add dev ath10 parent 1:0 prio 1 u32 match mark 25600 0xff00 classid 1:1
  ebtables -t nat -A POSTROUTING -o ath10 -j mark --mark-or 25600
  
  tc qdisc add dev ath10 parent 1:1 handle 2: htb
  tc class add dev ath10 parent 2: classid 2:1 htb rate 1000kbit ceil 1200kbit burst 1000kbit cburst 1000kbit mtu 10000
  
  tc filter add dev ath10 parent 2: prio 1 handle 800::800 u32 match u16 0x0800 0xffff at -2 match u32 0xb414701c 0xffffffff at -12 match u16 0xa088 0xffff at -14 flowid 2:1
  tc filter add dev ath10 parent 2: prio 1 handle 800::c00 u32 match u32 0xb414701c 0xffffffff at -20 match u16 0xa088 0xffff at -22 flowid 2:1
  

• 入口流限制
  

  ifconfig imq0 txqueuelen 32 up
  tc qdisc add dev imq0 handle 1: root htb
  tc class add dev imq0 parent 1: classid 1:1 htb rate 2000kbit mtu 10000
  tc filter add dev imq0 parent 1:0 prio 1 u32 match mark 25600 0xff00 classid 1:1
  iptables -A PREROUTING -t mangle -m physdev --physdev-in ath10 -j MARK --or-mark 25600
  iptables -t mangle -A PREROUTING -m physdev --physdev-in ath10 -j IMQ --todev 0
  
  tc qdisc add dev imq0 parent 1:1 handle 2: htb
  tc class add dev imq0 parent 2: classid 2:1 htb rate 2000kbit ceil 2400kbit burst 1000kbit cburst 1000kbit mtu 10000
  
  tc filter add dev imq0 parent 2: prio 1 handle 800::800 u32 match u16 0x0800 0xffff at -2 match u16 0x701c 0xffff at -4 match u32 0xa088b414 0xffffffff at -8 flowid 2:1
  tc filter add dev imq0 parent 2: prio 1 handle 800::c00 u32 match u16 0x701c 0xffff at -12 match u32 0xa088b414 0xffffffff at -16 flowid 2:1
  

• plain simple bandwidth sharing (aka traffic shaping) with HTB
  

  #!/bin/sh
  # We have 1000kbit upload and want to guarantee each user a certain amount of it.
  # If one user does not use its full quota, the unused quota get evenly distributed amongst the other users.
  
  # Variables
  IF_DSL=pppoe-dsl
  TC=$(which tc)
  IPT=$(which iptables)
  IPTMOD="$IPT -t mangle -A POSTROUTING -o $IF_DSL"
  IP_USER1=10.0.0.1
  IP_USER2=10.0.0.2
  IP_USER3=10.0.0.3
  IP_USER4=10.0.0.4
  
  insmod sch_htb
  
  $TC qdisc add dev $IF_DSL root       handle 1:    htb default 40
  $TC class add dev $IF_DSL parent 1:  classid 1:1  htb rate 1000kbit
  $TC class add dev $IF_DSL parent 1:1 classid 1:10 htb rate 250kbit #-- 25% to user1
  $TC class add dev $IF_DSL parent 1:1 classid 1:20 htb rate 250kbit #-- 25% to user2
  $TC class add dev $IF_DSL parent 1:1 classid 1:30 htb rate 350kbit #-- 35% to user3
  $TC class add dev $IF_DSL parent 1:1 classid 1:40 htb rate 150kbit #-- 15% to user4
  
  $IPTMOD -s $IP_USER1 -j CLASSIFY --set-class 1:10
  $IPTMOD -s $IP_USER2 -j CLASSIFY --set-class 1:20
  $IPTMOD -s $IP_USER3 -j CLASSIFY --set-class 1:30
  $IPTMOD -s $IP_USER4 -j CLASSIFY --set-class 1:40