Skip to content
ajdevhub
10 min read
VPP MASTERY · PHASE 4 · WEEKS 14–18
🔨 Plugin Development
Scaffold · Binary API (.api) · CLI · bihash classifier · Stateful tracker · Test framework
VLIB_REGISTER_NODE .api files VLIB_CLI_COMMAND vpptest Projects 6 & 7

PLUGIN SCAFFOLD AND FILE LAYOUT

🏗️

Generating and Understanding a Plugin

SCAFFOLD

VPP ships an Emacs/shell plugin generator that creates all boilerplate files. Always start here, then modify.

# Generate plugin scaffold
cd vpp
extras/emacs/make-plugin.sh
# Prompts for: plugin name (e.g. "classify")
# Generates: src/plugins/classify/

# Generated file layout:
src/plugins/classify/
├── CMakeLists.txt        # cmake build rules
├── classify.h            # main_t struct, per-worker data, extern declarations
├── classify.c            # plugin init, API message handlers
├── node.c                # the graph node function
├── classify.api          # binary API message definitions
├── classify_all_api_h.h  # generated: #include of .api.h files
├── classify_msg_enum.h   # generated: enum of message IDs
├── setup.pg              # packet generator script for testing
└── test/
    └── test_classify.py  # Python test using VppTestCase

Key structures in classify.h:

/* Plugin main struct - singleton, holds all plugin state */
typedef struct {
    /* Per-worker data - allocated as a vec, indexed by thread_index */
    classify_per_worker_t *per_worker;

<span class="cm">/* Global state (bihash tables, pool of rules, config) */</span>
clib_bihash_8_8_t flow_table;
classify_rule_t *rules;           <span class="cm">/* pool */</span>
<span class="ck">u32</span>  rule_count;

<span class="cm">/* vlib / vnet handles cached for fast access */</span>
vlib_main_t    *vlib_main;
vnet_main_t    *vnet_main;


} classify_main_t;


extern classify_main_t classify_main;


/* Per-worker struct - thread-local, no locking needed */
typedef struct {
u64  n_classified;
u64  n_passed;
u64  n_dropped;
} classify_per_worker_t;

⚙️

Plugin Init and Registration

LIFECYCLE
/* Plugin registration - VPP loads this .so at startup */
VLIB_PLUGIN_REGISTER () = {
    .version = VPP_BUILD_VER,
    .description = "Packet classifier plugin",
};

/* Init function - runs once after all plugins are loaded */
static clib_error_t *
classify_init (vlib_main_t *vm)
{
classify_main_t *cm = &classify_main;
cm->vlib_main  = vm;
cm->vnet_main  = vnet_get_main();


<span class="cm">/* Allocate per-worker structs */</span>
vec_validate_init_empty(cm->per_worker,
    vlib_num_workers(), (classify_per_worker_t){0});

<span class="cm">/* Init bihash - 64K buckets, 128MB backing */</span>
clib_bihash_init_8_8(&cm->flow_table, "classify-flow",
                     64 * 1024, 128 << 20);

<span class="cm">/* Register API message handlers */</span>
classify_api_hookup(vm);

<span class="ck">return</span> 0;


}
VLIB_INIT_FUNCTION (classify_init);


/* Config function - parses startup.conf stanza if any */
VLIB_CONFIG_FUNCTION (classify_config, “classify”);

BINARY API - .api FILES

📡

Defining Messages in .api Files

BINARY API

VPP's binary API is the programmatic control-plane interface - used by vppctl, GoVPP, vpp_papi, and any management agent. API messages are defined in .api files and compiled into C, Go, and Python stubs automatically.

/* classify.api - message definitions */

/* Option: API version */
option version = "1.0.0";
import "vnet/interface_types.api";

/* ── Add a classifier rule ── */
autoreply define classify_add_rule {
    u32  client_index;
    u32  context;
    vl_api_interface_index_t sw_if_index;  /* interface to apply rule on */
    u32  src_ip;                            /* host byte order */
    u32  dst_ip;
    u16  src_port;
    u16  dst_port;
    u8   protocol;
    u8   action;                            /* 0=pass, 1=drop, 2=redirect */
    u32  redirect_sw_if_index;
};

/* autoreply generates a _reply message automatically */
/* VPP sends: typedef classify_add_rule_reply_t { i32 retval; } */

/* ── Dump all rules (uses dump+details pattern) ── */
define classify_rule_dump {
    u32 client_index;
    u32 context;
};
define classify_rule_details {
    u32 context;
    u32 rule_index;
    vl_api_interface_index_t sw_if_index;
    u32 src_ip;
    u32 dst_ip;
    u16 src_port;
    u16 dst_port;
    u8  protocol;
    u8  action;
    u64 packet_count;
    u64 byte_count;
};

Implementing the handler in classify.c:

/* API message handler - called from the main thread */
static void
vl_api_classify_add_rule_t_handler (vl_api_classify_add_rule_t *mp)
{
    classify_main_t *cm = &classify_main;
    vl_api_classify_add_rule_reply_t *rmp;
    int rv = 0;

    /* Validate input */
    u32 sw_if_index = ntohl(mp->sw_if_index);
    if (!vnet_sw_interface_is_valid(vnet_get_main(), sw_if_index)) {
        rv = VNET_API_ERROR_INVALID_SW_IF_INDEX;
        goto done;
    }

    /* Add rule - hold a vlib barrier since we're modifying global state */
    vlib_worker_thread_barrier_sync(cm->vlib_main);
    rv = classify_add_rule_internal(cm, mp);
    vlib_worker_thread_barrier_release(cm->vlib_main);

done:
    /* Send reply */
    REPLY_MACRO(VL_API_CLASSIFY_ADD_RULE_REPLY);
}

/* Registration glue */
static void
classify_api_hookup (vlib_main_t *vm)
{
#define _(N,n) vl_msg_api_set_handlers(VL_API_##N, #n,
    vl_api_##n##_t_handler, /* ... */)
    foreach_classify_api_msg;
#undef _
}

CLI COMMANDS - VLIB_CLI_COMMAND

💻

Registering and Implementing CLI Commands

CLI
/* ── Registration ── */
VLIB_CLI_COMMAND (classify_add_rule_command, static) = {
    .path = "classify add rule",
    .short_help = "classify add rule <if> src <ip> dst <ip> [proto <N>] [drop|pass]",
    .function = classify_add_rule_command_fn,
};

VLIB_CLI_COMMAND (classify_show_command, static) = {
    .path = "show classify",
    .short_help = "show classify [interface <if>]",
    .function = classify_show_command_fn,
};

/* ── Implementation ── */
static clib_error_t *
classify_add_rule_command_fn (vlib_main_t *vm,
    unformat_input_t *input, vlib_cli_command_t *cmd)
{
    classify_main_t *cm = &classify_main;
    u32 sw_if_index = ~0;
    ip4_address_t src = {0}, dst = {0};
    u8  protocol = 0;
    int drop = 0;
    clib_error_t *error = 0;

    /* Parse arguments */
    while (unformat_check_input(input) != UNFORMAT_END_OF_INPUT) {
        if (unformat(input, "%U", unformat_vnet_sw_interface,
                      vnet_get_main(), &sw_if_index))
            ;
        else if (unformat(input, "src %U", unformat_ip4_address, &src))
            ;
        else if (unformat(input, "dst %U", unformat_ip4_address, &dst))
            ;
        else if (unformat(input, "proto %d", &protocol))
            ;
        else if (unformat(input, "drop"))
            drop = 1;
        else {
            error = clib_error_return(0, "Unknown argument: '%U'",
                                      format_unformat_error, input);
            goto done;
        }
    }

    if (sw_if_index == ~0) {
        error = clib_error_return(0, "Interface required");
        goto done;
    }

    classify_add_rule_internal(cm, sw_if_index, &src, &dst, protocol, drop);
    vlib_cli_output(vm, "Rule added (sw_if_index %d)\n", sw_if_index);

done:
    return error;
}

/* ── Show command ── */
static clib_error_t *
classify_show_command_fn (vlib_main_t *vm,
    unformat_input_t *input, vlib_cli_command_t *cmd)
{
    classify_main_t *cm = &classify_main;
    classify_rule_t *rule;
    u32 sw_if_index = ~0;

    if (unformat(input, "%U", unformat_vnet_sw_interface,
                  vnet_get_main(), &sw_if_index))
        ;

    /* Print header */
    vlib_cli_output(vm, "%-5s %-16s %-16s %-6s %-6s %-10s %-10s\n",
                    "ID", "SRC", "DST", "PROTO", "ACTION", "PACKETS", "BYTES");

    pool_foreach(rule, cm->rules) {
        if (sw_if_index != ~0 && rule->sw_if_index != sw_if_index)
            continue;
        vlib_cli_output(vm, "%-5d %-16U %-16U %-6d %-6s %-10llu %-10llu\n",
                        rule - cm->rules,
                        format_ip4_address, &rule->src,
                        format_ip4_address, &rule->dst,
                        rule->protocol,
                        rule->action == CLASSIFY_ACTION_DROP ? "DROP" : "PASS",
                        rule->n_packets, rule->n_bytes);
    }
    return 0;
}

COMPLETE CLASSIFIER NODE - BIHASH FAST PATH

🔎

5-Tuple Classifier Node Implementation

PROJECT 6
/* Error strings */
static char *classify_error_strings[] = {
#define _(n,s) s,
    foreach_classify_error
#undef _
};

VLIB_REGISTER_NODE (classify_node) = {
.name = “pkt-classify”,
.vector_size = sizeof(u32),
.type = VLIB_NODE_TYPE_INTERNAL,
.n_errors = CLASSIFY_N_ERROR,
.error_strings = classify_error_strings,
.n_next_nodes = CLASSIFY_N_NEXT,
.next_nodes = {
[CLASSIFY_NEXT_DROP]   = “error-drop”,
[CLASSIFY_NEXT_PASS]   = “ip4-lookup”,
},
.format_trace = format_classify_trace,
};


VLIB_NODE_FN (classify_node) (vlib_main_t *vm,
vlib_node_runtime_t *node, vlib_frame_t *frame)
{
classify_main_t *cm = &classify_main;
u32 thread_index = vm->thread_index;
classify_per_worker_t *pw = &cm->per_worker[thread_index];


<span class="ck">u32</span> n_left_from, *from;
from = vlib_frame_vector_args(frame);
n_left_from = frame->n_vectors;

<span class="ck">u16</span> nexts[VLIB_FRAME_SIZE];
<span class="ck">u16</span> *next = nexts;

<span class="cm">/* Quad loop */</span>
<span class="ck">while</span> (n_left_from >= 8) {
    vlib_buffer_t *b0, *b1, *b2, *b3;
    ip4_header_t *ip0, *ip1, *ip2, *ip3;
    clib_bihash_kv_8_8_t kv;

    vlib_prefetch_buffer_with_index(vm, from[4], LOAD);
    vlib_prefetch_buffer_with_index(vm, from[5], LOAD);
    vlib_prefetch_buffer_with_index(vm, from[6], LOAD);
    vlib_prefetch_buffer_with_index(vm, from[7], LOAD);

    vlib_get_buffers(vm, from, &b0, 4);

    ip0 = vlib_buffer_get_current(b0);
    ip1 = vlib_buffer_get_current(b1);
    ip2 = vlib_buffer_get_current(b2);
    ip3 = vlib_buffer_get_current(b3);

    <span class="cm">/* Macro: pack 5-tuple into u64 key for bihash_8_8 lookup */</span>


#define CLASSIFY_KEY(ip) </span>
((((u64)(ip)->src_address.as_u32) << 32) | (ip)->dst_address.as_u32)


    kv.key = CLASSIFY_KEY(ip0);
    next[0] = (clib_bihash_search_8_8(&cm->flow_table, &kv, &kv) == 0)
              ? (u16)kv.value : CLASSIFY_NEXT_PASS;

    kv.key = CLASSIFY_KEY(ip1);
    next[1] = (clib_bihash_search_8_8(&cm->flow_table, &kv, &kv) == 0)
              ? (u16)kv.value : CLASSIFY_NEXT_PASS;

    kv.key = CLASSIFY_KEY(ip2);
    next[2] = (clib_bihash_search_8_8(&cm->flow_table, &kv, &kv) == 0)
              ? (u16)kv.value : CLASSIFY_NEXT_PASS;

    kv.key = CLASSIFY_KEY(ip3);
    next[3] = (clib_bihash_search_8_8(&cm->flow_table, &kv, &kv) == 0)
              ? (u16)kv.value : CLASSIFY_NEXT_PASS;

    pw->n_classified += 4;
    from += 4; next += 4; n_left_from -= 4;
}

<span class="cm">/* Dual and single drain loops omitted for brevity - same pattern */</span>

vlib_buffer_enqueue_to_next(vm, node,
    vlib_frame_vector_args(frame), nexts, frame->n_vectors);
<span class="ck">return</span> frame->n_vectors;


}

STATEFUL CONNECTION TRACKER - PROJECT 7

🔗

Per-Flow State + Timeout Sweep

ADVANCED

A stateful tracker extends the classifier to maintain per-flow connection state (NEW, ESTABLISHED, FIN_WAIT, CLOSED), byte/packet counters per flow, and a timeout sweep to expire idle flows. This is the foundation of NAT, firewalls, and connection tracking.

/* Flow state machine */
typedef enum {
    FLOW_STATE_NEW = 0,
    FLOW_STATE_ESTABLISHED,
    FLOW_STATE_FIN_WAIT,
    FLOW_STATE_CLOSED,
} flow_state_t;

/* Flow entry - stored in a pool */
typedef struct {
    /* Key (also used as bihash lookup) */
    ip4_address_t src, dst;
    u16  src_port, dst_port;
    u8   protocol;

    /* State */
    flow_state_t state;
    f64  last_seen;          /* vlib_time_now() */
    u64  n_packets, n_bytes;

    /* Timer wheel handle */
    u32  timer_handle;
} flow_entry_t;

/* Fast path: per-packet state update */
VLIB_NODE_FN(flow_track_node)(vlib_main_t *vm, ...) {
    /* ... dual loop ... */
    f64 now = vlib_time_now(vm);

    /* Lookup flow */
    clib_bihash_kv_16_8_t kv;
    pack_5tuple(&kv.key, ip0, tcp0);

    if (PREDICT_TRUE(clib_bihash_search_16_8(&fm->flow_table, &kv, &kv) == 0)) {
        /* Existing flow */
        flow_entry_t *f = pool_elt_at_index(fm->flows, kv.value);
        f->last_seen = now;
        f->n_packets++;
        f->n_bytes += b0->current_length;

        /* State transition on TCP flags */
        if (tcp0->flags & TCP_FLAG_FIN)
            f->state = FLOW_STATE_FIN_WAIT;
        else if (f->state == FLOW_STATE_NEW)
            f->state = FLOW_STATE_ESTABLISHED;
    } else {
        /* New flow - send to slow path for allocation */
        next0 = FLOW_NEXT_SLOW_PATH;
    }
}

/* Timeout sweep - runs in a PROCESS node once per second */
VLIB_NODE_FN(flow_timeout_process)(vlib_main_t *vm, ...) {
    while (1) {
        vlib_process_suspend(vm, 1.0);  /* yield for 1 second */
        f64 now = vlib_time_now(vm);
        flow_entry_t *f;
        pool_foreach(f, fm->flows) {
            if (now - f->last_seen > FLOW_TIMEOUT_SEC) {
                /* Remove from bihash + free pool slot */
                flow_delete(fm, f);
            }
        }
    }
}

💡 Thread safety in the tracker: The fast path (flow lookup + counter update) runs on worker threads. The timeout sweep (pool_foreach + flow_delete) runs on the main thread. To safely delete flows from a worker-accessed bihash, use vlib_worker_thread_barrier_sync in the sweep, or use atomic operations / a lock-free delete queue. The barrier approach is simpler and correct for low-frequency deletions.

VPP TEST FRAMEWORK

🧪

Writing Python Tests with VppTestCase

TESTING

VPP's test framework (test/) provides VppTestCase, a Python unittest subclass that spins up a real VPP instance, sends packets via the packet generator or raw sockets, and makes assertions on counters and captured packets.

## test/test_classify.py
from framework import VppTestCase
from scapy.layers.inet import IP, TCP, UDP, Ether
from vpp_papi import VppEnum

class TestClassify(VppTestCase):
    """Packet Classifier Plugin Tests"""

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()

    def setUp(self):
        super().setUp()
        # Create two loopback interfaces for testing
        self.create_loopback_interfaces(2)
        self.lo0, self.lo1 = self.lo_interfaces

        for i in self.lo_interfaces:
            i.admin_up()
            i.config_ip4()
            i.resolve_arp()

        # Enable classify feature on lo0
        self.vapi.classify_enable_disable(
            sw_if_index=self.lo0.sw_if_index,
            enable=1
        )

    def tearDown(self):
        for i in self.lo_interfaces:
            i.unconfig_ip4()
            i.admin_down()
        super().tearDown()

    def test_drop_rule(self):
        """Verify DROP rule drops matching packets"""
        # Add DROP rule for src 10.0.0.100 → dst 10.0.0.200
        self.vapi.classify_add_rule(
            sw_if_index=self.lo0.sw_if_index,
            src_ip=socket.inet_aton("10.0.0.100"),
            dst_ip=socket.inet_aton("10.0.0.200"),
            action=1  # DROP
        )

        # Craft and send matching packet
        pkts = [Ether() / IP(src="10.0.0.100", dst="10.0.0.200") / TCP()]
        self.send_and_assert_no_replies(self.lo0, pkts)

        # Verify drop counter incremented
        stats = self.vapi.classify_stats_get()
        self.assertEqual(stats.n_dropped, 1)

    def test_pass_rule(self):
        """Verify non-matching packets pass through"""
        pkts = [Ether() / IP(src="10.0.1.1", dst="10.0.1.2") / UDP()]
        self.send_and_expect(self.lo0, pkts, self.lo1)

Run your test: make test TEST=test_classify

P4 COMPLETION CHECKLIST

  • Can generate a plugin scaffold and explain the purpose of each generated file
  • Can write classify_main_t, classify_per_worker_t correctly - global vs per-worker data
  • Know the VLIB_PLUGIN_REGISTER + VLIB_INIT_FUNCTION lifecycle
  • Understand how to use vlib_worker_thread_barrier_sync when modifying global state from API handlers
  • Can write a complete .api file with autoreply define and dump/details pair
  • Can implement an API message handler: parse, validate, REPLY_MACRO
  • Can write VLIB_CLI_COMMAND with unformat_input_t parsing and vlib_cli_output printing
  • Can implement a bihash-based classifier node using the dual-loop pattern
  • Understand the stateful tracker design: pool for flow entries, bihash for fast lookup, PROCESS node for timeout sweep
  • Can write a VppTestCase Python test: setUp with loopbacks, send_and_assert_no_replies, counter assertions
  • Know how to run a specific test: make test TEST=test_mytest
  • Completed Project 6 (bihash classifier) and Project 7 (stateful tracker)

✅ Phase 4 complete. You can now build production VPP plugins end-to-end. Move to Phase 5 - Control Plane & GoVPP to learn how to drive VPP programmatically from Go.

← TAP · AF_XDP 🗺️ Roadmap Next: Control Plane →