使用ebpf 监控golang 应用

一、背景

使用ebpf 监控grpc-go的应用，grpc-go http2 client的处理点

1.  
    
2.  
   func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.HeaderField, onWrite func()) error {
3.  
   ......
4.  
   }
5.  
    
6.  
   // operateHeaders takes action on the decoded headers.
7.  
   func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
8.  
   }

使用ebpf 监控埋点:

1.  
   SEC("uprobe/谷歌.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
2.  
   int uprobe__probe_http2_client_operate_headers(struct pt_regs* ctx) {
3.  
   }
4.  
    

运行流程:

我们在 uprobe__probe_http2_client_operate_headers 里做一些处理，统计grpc的数据，进行遥测。编写代码过程中我们需要关注的一些事情

1、golang 程序中 t *http2Client 和  frame *http2.MetaHeadersFrame 在寄存器中的位置是如何分配的？

1.  
   type MetaHeadersFrame struct {
2.  
   *HeadersFrame
3.  
    
4.  
   // Fields are the fields contained in the HEADERS and
5.  
   // CONTINUATION frames. The underlying slice is owned by the
6.  
   // Framer and must not be retained after the next call to
7.  
   // ReadFrame.
8.  
   //
9.  
   // Fields are guaranteed to be in the correct http2 order and
10.  
    // not have unknown pseudo header fields or invalid header
11.  
    // field names or values. Required pseudo header fields may be
12.  
    // missing, however. Use the MetaHeadersFrame.Pseudo accessor
13.  
    // method access pseudo headers.
14.  
    Fields []hpack.HeaderField
15.  
     
16.  
    // Truncated is whether the max header list size limit was hit
17.  
    // and Fields is incomplete. The hpack decoder state is still
18.  
    // valid, however.
19.  
    Truncated bool
20.  
    }

2、http2Client 和 MetaHeadersFrame 结构体中各个成员的偏移量如何确定？

1.  
   type http2Client struct {
2.  
   lastRead int64 // Keep this field 64-bit aligned. Accessed atomically.
3.  
   ctx context.Context
4.  
   cancel context.CancelFunc
5.  
   ctxDone <-chan struct{} // Cache the ctx.Done() chan.
6.  
   userAgent string
7.  
   // address contains the resolver returned address for this transport.
8.  
   // If the `ServerName` field is set, it takes precedence over `CallHdr.Host`
9.  
   // passed to `NewStream`, when determining the :authority header.
10.  
    address resolver.Address
11.  
    md metadata.MD
12.  
    .....
13.  
    }

 以 MetaHeadersFrame 为例子

 我们在开发中需要找到 正确的offset 才能正确的取到内存。

二、我们需要做什么？

1、如何确定参数在寄存器中的位置？

我们使用llvm-dwarfdump-14 查看二进制中dwarf 构成:

llvm-dwarfdump-14 grpc-client |grep "loopyWriter).writeHeader" -C 100

返回格式:

1.  
   0x0018c375: DW_TAG_subprogram
2.  
                   DW_AT_name ("谷歌.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
3.  
                   DW_AT_low_pc (0x00000000007139a0)
4.  
                   DW_AT_high_pc (0x0000000000713df2)
5.  
                   DW_AT_frame_base (DW_OP_call_frame_cfa)
6.  
                   DW_AT_decl_file ("/home/zhanglei/data/grpc-demo/vendor/谷歌.golang.org/grpc/internal/transport/controlbuf.go")
7.  
                   DW_AT_external (0x01)
8.  
    
9.  
   0x0018c3d2: DW_TAG_formal_parameter
10.  
                      DW_AT_name ("l")
11.  
                      DW_AT_variable_parameter (0x00)
12.  
                      DW_AT_decl_line (677)
13.  
                      DW_AT_type (0x00000000000e6bc2 "谷歌.golang.org/grpc/internal/transport.loopyWriter *")
14.  
                      DW_AT_location (0x001a603f:
15.  
                         [0x00000000007139a0, 0x00000000007139fc): DW_OP_reg0 RAX
16.  
                         [0x00000000007139fc, 0x0000000000713df2): DW_OP_call_frame_cfa)
17.  
     
18.  
    0x0018c3e0: DW_TAG_formal_parameter
19.  
                      DW_AT_name ("streamID")
20.  
                      DW_AT_variable_parameter (0x00)
21.  
                      DW_AT_decl_line (677)
22.  
                      DW_AT_type (0x0000000000082ad4 "uint32")
23.  
                      DW_AT_location (0x001a6085:
24.  
                         [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg3 RBX
25.  
                         [0x0000000000713a02, 0x0000000000713df2): DW_OP_fbreg 8)
26.  
     
27.  
    0x0018c3f5: DW_TAG_formal_parameter
28.  
                      DW_AT_name ("endStream")
29.  
                      DW_AT_variable_parameter (0x00)
30.  
                      DW_AT_decl_line (677)
31.  
                      DW_AT_type (0x0000000000082b87 "bool")
32.  
                      DW_AT_location (0x001a60cc:
33.  
                         [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg2 RCX
34.  
                         [0x0000000000713a02, 0x0000000000713df2): DW_OP_fbreg 12)
35.  
     
36.  
    0x0018c40b: DW_TAG_formal_parameter
37.  
                      DW_AT_name ("hf")
38.  
                      DW_AT_variable_parameter (0x00)
39.  
                      DW_AT_decl_line (677)
40.  
                      DW_AT_type (0x00000000000e2826 "[]golang.org/x/net/http2/hpack.HeaderField")
41.  
                      DW_AT_location (0x001a6113:
42.  
                         [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x8, DW_OP_reg8 R8, DW_OP_piece 0x8
43.  
                         [0x0000000000713a02, 0x0000000000713a4e): DW_OP_fbreg 16, DW_OP_piece 0x8, DW_OP_fbreg 24, DW_OP_piece 0x8, DW_OP_piece 0x8
44.  
                         [0x0000000000713a4e, 0x0000000000713df2): DW_OP_piece 0x8, DW_OP_fbreg 24, DW_OP_piece 0x8, DW_OP_piece 0x8)
45.  
     
46.  
    0x0018c41a: DW_TAG_formal_parameter
47.  
                      DW_AT_name ("onWrite")
48.  
                      DW_AT_variable_parameter (0x00)
49.  
                      DW_AT_decl_line (677)
50.  
                      DW_AT_type (0x00000000000844f2 "func()")
51.  
                      DW_AT_location (0x001a6184:
52.  
                         [0x00000000007139a0, 0x0000000000713a02): DW_OP_reg9 R9)

数据格式简介:

我们可以找到func (l *loopyWriter) writeHeader 的参数的所在寄存器的位置

1.  
   DW_AT_name ("l") 的 [0x00000000007139a0, 0x00000000007139fc): DW_OP_reg0 RAX 在寄存器0 上
2.  
   DW_AT_name ("streamID") : DW_OP_reg3 RBX 在寄存器3上
3.  
   DW_AT_name ("endStream") DW_OP_reg2 RCX 在寄存器2上
4.  
   DW_AT_name ("hf") 在寄存器5 上 DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x8, DW_OP_reg8 R8, DW_OP_piece 0x8
5.  
   DW_AT_name ("onWrite") DW_OP_reg9 R9 在寄存器9上

2、如何定位结构体中的成员

1.  
   0x000e6c07: DW_TAG_structure_type
2.  
                   DW_AT_name ("谷歌.golang.org/grpc/internal/transport.loopyWriter")
3.  
                   DW_AT_byte_size (88)
4.  
                   DW_AT_GO_kind (0x19)
5.  
                   DW_AT_GO_runtime_type (0x00000000000b9a20)
6.  
    
7.  
   0x000e6c48: DW_TAG_member
8.  
                     DW_AT_name ("side")
9.  
                     DW_AT_data_member_location (0)
10.  
                      DW_AT_type (0x00000000000e5e96 "谷歌.golang.org/grpc/internal/transport.side")
11.  
                      DW_AT_GO_embedded_field (0x00)
12.  
     
13.  
    0x000e6c54: DW_TAG_member
14.  
                      DW_AT_name ("cbuf")
15.  
                      DW_AT_data_member_location (8)
16.  
                      DW_AT_type (0x00000000000e681d "谷歌.golang.org/grpc/internal/transport.controlBuffer *")
17.  
                      DW_AT_GO_embedded_field (0x00)
18.  
     
19.  
    0x000e6c60: DW_TAG_member
20.  
                      DW_AT_name ("sendQuota")
21.  
                      DW_AT_data_member_location (16)
22.  
                      DW_AT_type (0x0000000000082ad4 "uint32")
23.  
                      DW_AT_GO_embedded_field (0x00)
24.  
     
25.  
    0x000e6c71: DW_TAG_member
26.  
                      DW_AT_name ("oiws")
27.  
                      DW_AT_data_member_location (20)
28.  
                      DW_AT_type (0x0000000000082ad4 "uint32")
29.  
                      DW_AT_GO_embedded_field (0x00)
30.  
     
31.  
    0x000e6c7d: DW_TAG_member
32.  
                      DW_AT_name ("estdStreams")
33.  
                      DW_AT_data_member_location (24)
34.  
                      DW_AT_type (0x00000000000e6d3c "map[uint32]*谷歌.golang.org/grpc/internal/transport.outStream")
35.  
                      DW_AT_GO_embedded_field (0x00)
36.  
     
37.  
    0x000e6c90: DW_TAG_member
38.  
                      DW_AT_name ("activeStreams")
39.  
                      DW_AT_data_member_location (32)
40.  
                      DW_AT_type (0x00000000000e6eaa "谷歌.golang.org/grpc/internal/transport.outStreamList *")
41.  
                      DW_AT_GO_embedded_field (0x00)
42.  
     
43.  
    0x000e6ca5: DW_TAG_member
44.  
                      DW_AT_name ("framer")
45.  
                      DW_AT_data_member_location (40)
46.  
                      DW_AT_type (0x00000000000e6f8a "谷歌.golang.org/grpc/internal/transport.framer *")
47.  
                      DW_AT_GO_embedded_field (0x00)

 以 loopyWriter 结构体为例子:

1.  
   type loopyWriter struct {
2.  
   side side
3.  
   cbuf *controlBuffer
4.  
   sendQuota uint32
5.  
   oiws uint32 // outbound initial window size.
6.  
   // estdStreams is map of all established streams that are not cleaned-up yet.
7.  
   // On client-side, this is all streams whose headers were sent out.
8.  
   // On server-side, this is all streams whose headers were received.
9.  
   estdStreams map[uint32]*outStream // Established streams.
10.  
    // activeStreams is a linked-list of all streams that have data to send and some
11.  
    // stream-level flow control quota.
12.  
    // Each of these streams internally have a list of data items(and perhaps trailers
13.  
    // on the server-side) to be sent out.
14.  
    activeStreams *outStreamList
15.  
    framer *framer
16.  
    hBuf *bytes.Buffer // The buffer for HPACK encoding.
17.  
    hEnc *hpack.Encoder // HPACK encoder.
18.  
    bdpEst *bdpEstimator
19.  
    draining bool
20.  
     
21.  
    // Side-specific handlers
22.  
    ssGoAwayHandler func(*goAway) (bool, error)
23.  
    }

side 的偏移量是  0 

cbuf 是 8 

sendQuota 是 16 

oiws 是 20

estdStreams 是 24

activeStreams 是 32 

framer 是 40

3、如何确定接口类型地址

nm grpc-client|grep "TCPConn,net.Conn"

000000000093d280 R go.itab.*net.TCPConn,net.Conn

 发现对应的符号表地址:

1.  
   nm grpc-server|grep "TCPConn,net.Conn"
2.  
   000000000093bfc0 R go.itab.*net.TCPConn,net.Conn

4、使用gdb 证明这些数据

1.  
   (gdb) i args
2.  
   l = 0xc00011c060
3.  
   streamID = 1
4.  
   endStream = false
5.  
   hf = {array = 0xc00007e870, len = 2, cap = 2}
6.  
   onWrite = {void (void)} 0x0
7.  
   ~r0 = <optimised out>
8.  
   (gdb) i r
9.  
   rax 0xc00011c060 824634884192
10.  
    rbx 0x1 1
11.  
    rcx 0x0 0
12.  
    rdx 0xc0001b29c0 824635500992
13.  
    rsi 0x2 2
14.  
    rdi 0xc00007e870 824634239088
15.  
    rbp 0xc000129ec8 0xc000129ec8
16.  
    rsp 0xc000129e78 0xc000129e78
17.  
    r8 0x2 2
18.  
    r9 0xc000021f80 824633859968
19.  
    r10 0xc00007e8c0 824634239168
20.  
    r11 0x1 1
21.  
    r12 0x1 1
22.  
    r13 0xffffffffffffffff -1
23.  
    r14 0xc00019d520 824635413792
24.  
    r15 0x0 0
25.  
    rip 0x7160c0 0x7160c0 <谷歌.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader>
26.  
    eflags 0x246 [ PF ZF IF ]
27.  
    cs 0x33 51
28.  
    ss 0x2b 43
29.  
    ds 0x0 0
30.  
    es 0x0 0
31.  
    fs 0x0 0
32.  
    gs 0x0 0

发现l 位于寄存器 

l -> rax

streamID -> rbx

endStream -> rcx

hf -> rdi

接口类型:

1.  
   (gdb) p (*l.framer.writer).conn
2.  
   $8 = {tab = 0x93bfc0 <TCPConn,net.Conn>, data = 0xc000010290}

 发现对应的符号表地址:

1.  
   nm grpc-server|grep "TCPConn,net.Conn"
2.  
   000000000093bfc0 R go.itab.*net.TCPConn,net.Conn

三、发现golang应用

uprober 包含了 多个rule

每个rule 又包含Selector和 Checker

Selector 用来根据版本选择对应的Hooker

Checker 用来检查 动态库或者二进制文件是否满足匹配规则

 

运行流程：

 

四、golang 注册ebpf 钩子

 

五.使用偏移量获取golang变量

定义 location 结构体:

1.  
   typedef struct {
2.  
   __s64 stack_offset; // 栈上的偏移量
3.  
   __s64 _register; // 寄存器的偏移量
4.  
   __u8 in_register; //是否在寄存器上
5.  
   __u8 exists;
6.  
   } location_t;
7.  
    
8.  
   // golang 切片
9.  
   typedef struct {
10.  
    location_t ptr;
11.  
    location_t len;
12.  
    location_t cap;
13.  
    } slice_location_t;

golang 参数变量在寄存器的布局:

1.  
   // This function was adapted from https://github.com/go-delve/delve:
2.  
   // - https://github.com/go-delve/delve/blob/cd9e6c02a6ca5f0d66c1f770ee10a0d8f4419333/pkg/proc/internal/ebpf/bpf/trace.bpf.c#L43
3.  
   // which is licensed under MIT.
4.  
   static __always_inline int read_register(struct pt_regs* ctx, int64_t regnum, void* dest) {
5.  
   // This volatile temporary variable is need when building with clang-14,
6.  
   // or the verifier will complain that we dereference a modified context
7.  
   // pointer.
8.  
   //
9.  
   // What happened in this case, is that the compiler tried to be smart by
10.  
    // incrementing the context pointer, before jumping to code that will
11.  
    // copy the value pointed to by the new pointer to `dest`. The generated
12.  
    // code looked like this:
13.  
    //
14.  
    // r1 = 40 // Increment the ptr
15.  
    // goto 3 <LBB0_9> // goto __builtin_memcpy
16.  
    //
17.  
    // What the memcpy does is deference the resulting pointer to get the
18.  
    // CPU register value (that’s where the bug was), then put it in the
19.  
    // dest location:
20.  
    //
21.  
    // r1 = *(u64 *)(r1 0) // BUG: Get the register value.
22.  
    // // This is the "modified context pointer"
23.  
    // *(u64 *)(r3 0) = r1 // Put it in dest
24.  
    //
25.  
    // By incrementing the pointer before dereferencing it, the verifier no
26.  
    // longer considering r1 to be a pointer to the context, but as a
27.  
    // pointer to some random memory address (even though it is in the
28.  
    // memory the range of the context struct).
29.  
    //
30.  
    // What we want the compiler to generate is something like this:
31.  
    //
32.  
    // // Switch branch:
33.  
    // r1 = *(u64 *)(r1 40) // read value to tmp var
34.  
    // goto 30 <LBB0_39> // goto *dest = tmp
35.  
    //
36.  
    // // *dest = tmp
37.  
    // *(u64 *)(r3 0) = r1
38.  
    //
39.  
    // This volatile `tmp` variable makes the compiler generate the code above.
40.  
    volatile u64 tmp = 0;
41.  
    switch (regnum) {
42.  
    case 0: // RAX
43.  
    tmp = ctx->ax;
44.  
    break;
45.  
    case 1: // RDX
46.  
    tmp = ctx->dx;
47.  
    break;
48.  
    case 2: // RCX
49.  
    tmp = ctx->cx;
50.  
    break;
51.  
    case 3: // RBX
52.  
    tmp = ctx->bx;
53.  
    break;
54.  
    case 4: // RSI
55.  
    tmp = ctx->si;
56.  
    break;
57.  
    case 5: // RDI
58.  
    tmp = ctx->di;
59.  
    break;
60.  
    case 6: // RBP
61.  
    tmp = ctx->bp;
62.  
    break;
63.  
    case 7: // RSP
64.  
    tmp = ctx->sp;
65.  
    break;
66.  
    case 8: // R8
67.  
    tmp = ctx->r8;
68.  
    break;
69.  
    case 9: // R9
70.  
    tmp = ctx->r9;
71.  
    break;
72.  
    case 10: // R10
73.  
    tmp = ctx->r10;
74.  
    break;
75.  
    case 11: // R11
76.  
    tmp = ctx->r11;
77.  
    break;
78.  
    case 12: // R12
79.  
    tmp = ctx->r12;
80.  
    break;
81.  
    case 13: // R13
82.  
    tmp = ctx->r13;
83.  
    break;
84.  
    case 14: // R14
85.  
    tmp = ctx->r14;
86.  
    break;
87.  
    case 15: // R15
88.  
    tmp = ctx->r15;
89.  
    break;
90.  
    default:
91.  
    return 1;
92.  
    }
93.  
    *(u64*)dest = tmp;
94.  
    return 0;
95.  
    }

 读取偏移量：

1.  
   static __always_inline int read_location(struct pt_regs* ctx, location_t* loc, size_t size, void* dest) {
2.  
   if (!loc->exists) {
3.  
   return 0;
4.  
   }
5.  
    
6.  
   if (loc->in_register) {
7.  
   if (size != REG_SIZE) {
8.  
   return 1;
9.  
   }
10.  
     
11.  
    return read_register(ctx, loc->_register, dest);
12.  
    } else {
13.  
    return read_stack(ctx, loc->stack_offset, size, dest);
14.  
    }
15.  
    }

1.  
   static __always_inline int read_stack(struct pt_regs* ctx, int64_t stack_offset, size_t size, void* dest) {
2.  
   // `ctx->sp` is correct for both x86_64 and ARM64
3.  
   uintptr_t stack_pointer = (uintptr_t) ctx->sp;
4.  
   uintptr_t address = stack_pointer stack_offset;
5.  
   return bpf_probe_read_user(dest, size, (void*) address);
6.  
   }

举一个例子:

1.  
   SEC("uprobe/谷歌.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
2.  
   int uprobe__probe_loopy_writer_write_header(struct pt_regs* ctx) {
3.  
   return 0;
4.  
   }

 从map读取偏移量:

1.  
   uint32_t pid = bpf_get_current_pid_tgid() >> 32;
2.  
   go_http2_symaddrs_t* symaddrs = bpf_map_lookup_elem(&http2_symaddrs_map, &pid);
3.  
   if (symaddrs == NULL) {
4.  
   log_trace("uprobe__probe_loopy_writer_write_header:writeHeader:symaddrs is NULL\n");
5.  
   return 0;
6.  
   }

 偏移量存储结构体:

1.  
   typedef struct {
2.  
   // ---- itable symbols ----
3.  
    
4.  
   // io.Writer interface types.
5.  
   __s64 http_http2bufferedWriter; // "go.itab.*net/http.http2bufferedWriter,io.Writer
6.  
   __s64 transport_bufWriter; // "谷歌.golang.org/grpc/internal/transport.bufWriter,io.Writer
7.  
    
8.  
   // ---- function argument locations ----
9.  
    
10.  
    // Arguments of net/http.(*http2Framer).WriteDataPadded.
11.  
    location_t http2Framer_WriteDataPadded_f_loc; // 8
12.  
    location_t http2Framer_WriteDataPadded_streamID_loc; // 16
13.  
    location_t http2Framer_WriteDataPadded_endStream_loc; // 20
14.  
    location_t http2Framer_WriteDataPadded_data_ptr_loc; // 24
15.  
    location_t http2Framer_WriteDataPadded_data_len_loc; // 32
16.  
     
17.  
    // Arguments of golang.org/x/net/http2.(*Framer).WriteDataPadded.
18.  
    location_t http2_WriteDataPadded_f_loc; // 8
19.  
    location_t http2_WriteDataPadded_streamID_loc; // 16
20.  
    location_t http2_WriteDataPadded_endStream_loc; // 20
21.  
    location_t http2_WriteDataPadded_data_ptr_loc; // 24
22.  
    location_t http2_WriteDataPadded_data_len_loc; // 32
23.  
     
24.  
    // Arguments of net/http.(*http2Framer).checkFrameOrder.
25.  
    location_t http2Framer_checkFrameOrder_fr_loc; // 8
26.  
    location_t http2Framer_checkFrameOrder_f_loc; // 16
27.  
     
28.  
    // Arguments of golang.org/x/net/http2.(*Framer).checkFrameOrder.
29.  
    location_t http2_checkFrameOrder_fr_loc; // 8
30.  
    location_t http2_checkFrameOrder_f_loc; // 16
31.  
     
32.  
    // Arguments of net/http.(*http2writeResHeaders).writeFrame.
33.  
    location_t writeFrame_w_loc; // 8
34.  
    location_t writeFrame_ctx_loc; // 16
35.  
     
36.  
    // Arguments of golang.org/x/net/http2/hpack.(*Encoder).WriteField.
37.  
    location_t WriteField_e_loc; // 8
38.  
    // Note that the HeaderField `f` is further broken down to its name and value members.
39.  
    // This is done so we can better control the location of these members from user-space.
40.  
    // In theory, there could be an ABI that splits these two members across stack and registers.
41.  
    location_t WriteField_f_name_loc; // 16
42.  
    location_t WriteField_f_value_loc; // 32
43.  
     
44.  
    // Arguments of net/http.(*http2serverConn).processHeaders.
45.  
    location_t processHeaders_sc_loc; // 8
46.  
    location_t processHeaders_f_loc; // 16
47.  
     
48.  
    // Arguments of 谷歌.golang.org/grpc/internal/transport.(*http2Server).operateHeaders.
49.  
    location_t http2Server_operateHeaders_t_loc; // 8
50.  
    location_t http2Server_operateHeaders_frame_loc; // 16
51.  
     
52.  
    // Arguments of 谷歌.golang.org/grpc/internal/transport.(*http2Client).operateHeaders.
53.  
    location_t http2Client_operateHeaders_t_loc; // 8
54.  
    location_t http2Client_operateHeaders_frame_loc; // 16
55.  
     
56.  
    // Arguments of 谷歌.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader.
57.  
    location_t writeHeader_l_loc; // 8
58.  
    location_t writeHeader_streamID_loc; // 16
59.  
    location_t writeHeader_endStream_loc; // 20
60.  
    slice_location_t writeHeader_hf_loc; // 24
61.  
     
62.  
    // ---- struct member offsets ----
63.  
     
64.  
    // Struct member offsets.
65.  
    // Naming maintains golang style: <struct>_<member>_offset
66.  
    // Note: values in comments represent known offsets, in case we need to fall back.
67.  
    // Eventually, they should be removed, because they are not reliable.
68.  
     
69.  
    // Members of golang.org/x/net/http2/hpack.HeaderField.
70.  
    int32_t HeaderField_Name_offset; // 0
71.  
    int32_t HeaderField_Value_offset; // 16
72.  
     
73.  
    // Members of 谷歌.golang.org/grpc/internal/transport.http2Server.
74.  
    int32_t http2Server_conn_offset; // 16 or 24
75.  
     
76.  
    // Members of 谷歌.golang.org/grpc/internal/transport.http2Client.
77.  
    int32_t http2Client_conn_offset; // 64
78.  
     
79.  
    // Members of 谷歌.golang.org/grpc/internal/transport.loopyWriter.
80.  
    int32_t loopyWriter_framer_offset; // 40
81.  
     
82.  
    // Members of golang.org/x/net/net/http2.Framer.
83.  
    int32_t Framer_w_offset; // 112
84.  
     
85.  
    // Members of golang.org/x/net/http2.MetaHeadersFrame.
86.  
    int32_t MetaHeadersFrame_HeadersFrame_offset; // 0
87.  
    int32_t MetaHeadersFrame_Fields_offset; // 0
88.  
     
89.  
    // Members of golang.org/x/net/http2.HeadersFrame.
90.  
    int32_t HeadersFrame_FrameHeader_offset; // 0
91.  
     
92.  
    // Members of golang.org/x/net/http2.FrameHeader.
93.  
    int32_t FrameHeader_Type_offset; // 1
94.  
    int32_t FrameHeader_Flags_offset; // 2
95.  
    int32_t FrameHeader_StreamID_offset; // 8
96.  
     
97.  
    // Members of golang.org/x/net/http2.DataFrame.
98.  
    int32_t DataFrame_data_offset; // 16
99.  
     
100.  
     // Members of 谷歌.golang.org/grpc/internal/transport.bufWriter.
101.  
     int32_t bufWriter_conn_offset; // 40
102.  
      
103.  
     // Members of net/http.http2serverConn.
104.  
     int32_t http2serverConn_conn_offset; // 16
105.  
     int32_t http2serverConn_hpackEncoder_offset; // 360
106.  
      
107.  
     // Members of net/http.http2HeadersFrame
108.  
     int32_t http2HeadersFrame_http2FrameHeader_offset; // 0
109.  
      
110.  
     // Members of net/http.http2FrameHeader.
111.  
     int32_t http2FrameHeader_Type_offset; // 1
112.  
     int32_t http2FrameHeader_Flags_offset; // 2
113.  
     int32_t http2FrameHeader_StreamID_offset; // 8
114.  
      
115.  
     // Members of golang.org/x/net/http2.DataFrame.
116.  
     int32_t http2DataFrame_data_offset; // 16
117.  
      
118.  
     // Members of net/http.http2writeResHeaders.
119.  
     int32_t http2writeResHeaders_streamID_offset; // 0
120.  
     int32_t http2writeResHeaders_endStream_offset; // 48
121.  
      
122.  
     // Members of net/http.http2MetaHeadersFrame.
123.  
     int32_t http2MetaHeadersFrame_http2HeadersFrame_offset; // 0
124.  
     int32_t http2MetaHeadersFrame_Fields_offset; // 8
125.  
      
126.  
     // Members of net/http.http2Framer.
127.  
     int32_t http2Framer_w_offset; // 112
128.  
      
129.  
     // Members of net/http.http2bufferedWriter
130.  
     int32_t http2bufferedWriter_w_offset; // 0
131.  
     } go_http2_symaddrs_t;

读取偏移量:

1.  
   void* loopy_writer_ptr = NULL;
2.  
   if (read_location(ctx, &symaddrs->writeHeader_l_loc,
3.  
   sizeof(loopy_writer_ptr), &loopy_writer_ptr)) {
4.  
   log_trace("uprobe__probe_loopy_writer_write_header:5\n");
5.  
   return 0;
6.  
   }

六、如何记录错误调试

1、打日志调试

1.  
   #define log_trace(fmt, ...) \
2.  
   ({ \
3.  
   char ____fmt[] = fmt; \
4.  
   bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
5.  
   })
6.  
    
7.  
   /* Macro to output debug logs to /sys/kernel/debug/tracing/trace_pipe
8.  
   */
9.  
   #if DEBUG == 1
10.  
    #define log_debug(fmt, ...) \
11.  
    ({ \
12.  
    char ____fmt[] = fmt; \
13.  
    bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
14.  
    })
15.  
    #else

使用:

log_debug("[grpc-c:lookup_version:]grpc-c version is not support;pid:%d;version:%d;\n", pid, *version);

2、记录metrics

主要记录行号和文件

1.  
   #ifndef TRACER_TELEMETRY
2.  
   #define TRACER_TELEMETRY
3.  
    
4.  
   #ifndef TRACER_TELEMETRY_KEY_LIMIT
5.  
   #define TRACER_TELEMETRY_KEY_LIMIT 4096
6.  
   #endif
7.  
    
8.  
   typedef struct {
9.  
   char file[TRACER_TELEMETRY_KEY_LIMIT];
10.  
    uint64_t line;
11.  
    } tracer_telemetry_key;
12.  
     
13.  
    BPF_HASH_MAP(tracer_telemetry, tracer_telemetry_key, uint64_t, 1024);
14.  
    BPF_PERCPU_ARRAY_MAP(tracer_telemetry_heap, __u32, tracer_telemetry_key, 1);
15.  
     
16.  
    static __always_inline tracer_telemetry_key* alloc_tracer_telemetry_key() {
17.  
    uint32_t kZero = 0;
18.  
    tracer_telemetry_key* value = bpf_map_lookup_elem(&tracer_telemetry_heap, &kZero);
19.  
    if (value == NULL) {
20.  
    return NULL;
21.  
    }
22.  
     
23.  
    value->line = 0;
24.  
    return value;
25.  
    }
26.  
     
27.  
    static __always_inline void increment_tracer_telemetry_count(const char* file, uint64_t line) {
28.  
    tracer_telemetry_key* key = alloc_tracer_telemetry_key();
29.  
    if (key == NULL) {
30.  
    return;
31.  
    }
32.  
     
33.  
    if (bpf_probe_read_str(&key->file, sizeof(key->file), file) == -1) {
34.  
    return;
35.  
    }
36.  
     
37.  
    key->line = line;
38.  
     
39.  
    uint64_t *val = NULL;
40.  
    val = bpf_map_lookup_elem(&tracer_telemetry, key);
41.  
    if (val == NULL) {
42.  
    log_debug("tracer:key->file:%s;file:%s\n", key->file, file);
43.  
    uint64_t tmp_value = 0;
44.  
    bpf_map_update_with_telemetry(tracer_telemetry, key, &tmp_value, BPF_NOEXIST);
45.  
    return;
46.  
    }
47.  
     
48.  
    val = bpf_map_lookup_elem(&tracer_telemetry, key);
49.  
    if (val == NULL) {
50.  
    return;
51.  
    }
52.  
     
53.  
    __sync_fetch_and_add(val, 1);
54.  
    }
55.  
     
56.  
    #ifndef INCR_TRACER_COUNT
57.  
    #define INCR_TRACER_COUNT increment_tracer_telemetry_count(__FILE__, __LINE__)
58.  
    #endif
59.  
     
60.  
    #endif

这篇好文章是转载于：学新通技术网