Support CFA based on general purpose registers

perforator/agent/collector/pkg/dso/bpf/unwindtable/gotest/ya.make

@@ -0,0 +1,3 @@
+GO_TEST_FOR(perforator/agent/collector/pkg/dso/bpf/unwindtable)
+
+END()

perforator/agent/collector/pkg/dso/bpf/unwindtable/manager.go

@@ -604,6 +604,32 @@ func (b *pageTableBuilder) flushNodes() error {
 // FIXME(sskvor): Generate this from the BTF
 const dwarfUnwindRuleUndefined = 0x7f
 
+// DWARF number of rbp, see the x86-64 psABI.
+const dwarfRegisterRBP = 6
+
+// isAtFramePointer reports whether the location is the address held in rbp itself.
+func isAtFramePointer(location *unwind.UnwindRule_RegisterPlusOffset) bool {
+	return location != nil && location.GetRegister() == dwarfRegisterRBP && location.GetOffset() == 0
+}
+
+// derefCfaRule converts CFA = *(reg + offset) + bias. The eBPF unwinder encodes the bias in the rule kind
+// and supports the two that occur in practice: 0 (gcc stack realignment) and 8 (openssl perlasm).
+func derefCfaRule(deref *unwind.UnwindRule_RegisterDerefOffset) unwinder.CfaUnwindRule {
+	rule := unwinder.CfaUnwindRule{
+		Regno:  uint8(deref.GetRegister()),
+		Offset: deref.GetOffset(),
+	}
+	switch deref.GetBias() {
+	case 0:
+		rule.Kind = unwinder.UnwindRuleRegisterDerefOffset
+	case 8:
+		rule.Kind = unwinder.UnwindRuleRegisterDerefOffsetPlus8
+	default:
+		rule.Kind = unwinder.UnwindRuleUnsupported
+	}
+	return rule
+}
+
 func fillRule(row row, page *unwinder.UnwindTablePageLeaf, idx int, pageStartPC uint64) {
 	rule := unwinder.UnwindRule{}
 
@@ -613,6 +639,8 @@ func fillRule(row row, page *unwinder.UnwindTablePageLeaf, idx int, pageStartPC
 		rule.Rbp = unwinder.RbpUnwindRule{Offset: int8(offset)}
 	} else if rbp != nil && rbp.GetCfaMinus8() != nil {
 		rule.Rbp = unwinder.RbpUnwindRule{Offset: int8(-8)}
+	} else if rbp != nil && rbp.GetDereference() && isAtFramePointer(rbp.GetRegisterOffset()) {
+		rule.Rbp = unwinder.RbpUnwindRule{Offset: int8(unwinder.DwarfUnwindRbpRuleAtRbp)}
 	} else {
 		rule.Rbp = unwinder.RbpUnwindRule{Offset: dwarfUnwindRuleUndefined}
 	}
@@ -626,6 +654,8 @@ func fillRule(row row, page *unwinder.UnwindTablePageLeaf, idx int, pageStartPC
 		rule.Cfa.Kind = unwinder.UnwindRuleRegisterOffset
 		rule.Cfa.Regno = uint8(unwinder.DwarfCfiStackRegisterNumber)
 		rule.Cfa.Offset = cfa.GetCfaPlusOffset().GetOffset()
+	} else if cfa != nil && cfa.GetRegisterDerefOffset() != nil {
+		rule.Cfa = derefCfaRule(cfa.GetRegisterDerefOffset())
 	} else {
 		rule.Cfa.Kind = unwinder.UnwindRuleUnsupported
 	}

perforator/agent/collector/pkg/dso/bpf/unwindtable/manager_test.go

@@ -0,0 +1,135 @@
+package unwindtable
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/yandex/perforator/perforator/agent/preprocessing/proto/unwind"
+	"github.com/yandex/perforator/perforator/internal/unwinder"
+)
+
+func protoDerefCfaRule(register uint32, offset, bias int32) *unwind.UnwindRule {
+	return &unwind.UnwindRule{Kind: &unwind.UnwindRule_RegisterDerefOffset_{RegisterDerefOffset: &unwind.UnwindRule_RegisterDerefOffset{
+		Register: register,
+		Offset:   offset,
+		Bias:     bias,
+	}}}
+}
+
+func unsupportedRule() *unwind.UnwindRule {
+	return &unwind.UnwindRule{Kind: &unwind.UnwindRule_Unsupported_{Unsupported: &unwind.UnwindRule_Unsupported{}}}
+}
+
+func testRow(cfa, rbp *unwind.UnwindRule) row {
+	return row{
+		table: &unwind.UnwindTable{
+			Dict: []*unwind.UnwindRule{
+				cfa,
+				{Kind: &unwind.UnwindRule_CfaMinus8{CfaMinus8: &unwind.UnwindRule_CFAMinus8{}}},
+				rbp,
+			},
+			StartPc: []uint64{0x1010},
+			PcRange: []uint64{0x20},
+			Cfa:     []uint32{0},
+			Ra:      []uint32{1},
+			Rbp:     []uint32{2},
+		},
+	}
+}
+
+func TestFillRuleDerefCfa(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		register uint32
+		offset   int32
+		bias     int32
+		expected unwinder.CfaUnwindRule
+	}{
+		{
+			name:     "openssl",
+			register: uint32(unwinder.DwarfCfiStackRegisterNumber),
+			offset:   152,
+			bias:     8,
+			expected: unwinder.CfaUnwindRule{
+				Kind:   unwinder.UnwindRuleRegisterDerefOffsetPlus8,
+				Regno:  uint8(unwinder.DwarfCfiStackRegisterNumber),
+				Offset: 152,
+			},
+		},
+		{
+			name:     "gcc-drap",
+			register: 6,
+			offset:   -8,
+			bias:     0,
+			expected: unwinder.CfaUnwindRule{
+				Kind:   unwinder.UnwindRuleRegisterDerefOffset,
+				Regno:  6,
+				Offset: -8,
+			},
+		},
+		{
+			name:     "unsupported-bias",
+			register: uint32(unwinder.DwarfCfiStackRegisterNumber),
+			offset:   16,
+			bias:     16,
+			expected: unwinder.CfaUnwindRule{
+				Kind:   unwinder.UnwindRuleUnsupported,
+				Regno:  uint8(unwinder.DwarfCfiStackRegisterNumber),
+				Offset: 16,
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			page := &unwinder.UnwindTablePageLeaf{}
+			fillRule(testRow(protoDerefCfaRule(tc.register, tc.offset, tc.bias), unsupportedRule()), page, 3, 0x1000)
+
+			require.Equal(t, tc.expected, page.Rules[3].Cfa)
+			require.Equal(t, int8(-8), page.Rules[3].Ra.Offset)
+			require.Equal(t, uint32(0x10), page.Pc[3])
+			require.Equal(t, uint32(0x20), page.Ranges[3])
+		})
+	}
+}
+
+func TestFillRuleRbpSavedAtRbp(t *testing.T) {
+	cfa := protoDerefCfaRule(6, -8, 0)
+	rbpAt := func(register uint32, offset int32, dereference bool) *unwind.UnwindRule {
+		return &unwind.UnwindRule{
+			Kind: &unwind.UnwindRule_RegisterOffset{RegisterOffset: &unwind.UnwindRule_RegisterPlusOffset{
+				Register: register,
+				Offset:   offset,
+			}},
+			Dereference: dereference,
+		}
+	}
+
+	for _, tc := range []struct {
+		name     string
+		rbp      *unwind.UnwindRule
+		expected int8
+	}{
+		{name: "gcc-drap", rbp: rbpAt(6, 0, true), expected: int8(unwinder.DwarfUnwindRbpRuleAtRbp)},
+		{name: "nonzero-offset", rbp: rbpAt(6, 8, true), expected: dwarfUnwindRuleUndefined},
+		{name: "other-register", rbp: rbpAt(7, 0, true), expected: dwarfUnwindRuleUndefined},
+		{name: "not-dereferenced", rbp: rbpAt(6, 0, false), expected: dwarfUnwindRuleUndefined},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			page := &unwinder.UnwindTablePageLeaf{}
+			fillRule(testRow(cfa, tc.rbp), page, 0, 0x1000)
+			require.Equal(t, tc.expected, page.Rules[0].Rbp.Offset)
+		})
+	}
+}
+
+func TestFillRuleRegisterCfa(t *testing.T) {
+	// openssl aesni_cbc_encrypt: CFA = r11 + 8. The eBPF unwinder resolves registers other than rsp and rbp.
+	cfa := &unwind.UnwindRule{Kind: &unwind.UnwindRule_RegisterOffset{RegisterOffset: &unwind.UnwindRule_RegisterPlusOffset{
+		Register: 11,
+		Offset:   8,
+	}}}
+
+	page := &unwinder.UnwindTablePageLeaf{}
+	fillRule(testRow(cfa, unsupportedRule()), page, 0, 0x1000)
+	require.Equal(t, unwinder.CfaUnwindRule{Kind: unwinder.UnwindRuleRegisterOffset, Regno: 11, Offset: 8}, page.Rules[0].Cfa)
+}

perforator/agent/collector/pkg/dso/bpf/unwindtable/ya.make

@@ -7,4 +7,8 @@ SRCS(
     row.go
 )
 
+GO_TEST_SRCS(manager_test.go)
+
 END()
+
+RECURSE_FOR_TESTS(gotest)

perforator/agent/collector/progs/unwinder/arch/arm/dwarf.h

@@ -36,6 +36,20 @@ static ALWAYS_INLINE void dwarf_unwind_setup_userspace_registers(
     cfi->lr = regs->lr;
 }
 
+static ALWAYS_INLINE void dwarf_unwind_setup_userspace_gprs(struct unwind_gprs* gprs, struct user_regs* regs) {
+    gprs->known = 0;
+}
+
+static ALWAYS_INLINE void dwarf_unwind_gprs_after_dwarf_step(struct unwind_gprs* gprs) {
+}
+
+static ALWAYS_INLINE void dwarf_unwind_gprs_forget(struct unwind_gprs* gprs) {
+}
+
+static ALWAYS_INLINE bool dwarf_unwind_gprs_get(struct unwind_gprs* gprs, u8 regno, u64* value) {
+    return false;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 // FIXME: we are fully dependendent on Link register here, but it might be invalidated

perforator/agent/collector/progs/unwinder/arch/arm/regs.h

@@ -17,6 +17,11 @@ struct user_regs {
     u64 lr;
 };
 
+// CFA based on registers other than sp and fp is not supported on arm64.
+struct unwind_gprs {
+    u16 known;
+};
+
 static ALWAYS_INLINE u64 regs_get_current_instruction(struct user_regs* regs) {
     return regs->pc;
 }

perforator/agent/collector/progs/unwinder/arch/x86/dwarf.h

@@ -35,6 +35,40 @@ static ALWAYS_INLINE void dwarf_unwind_setup_userspace_registers(
 
 ////////////////////////////////////////////////////////////////////////////////
 
+enum {
+    // Registers a callee may change without describing it in its CFI. DWARF treats registers without a rule
+    // as having the same value in the caller, and hand-written assembly relies on it: aesni_gcm_encrypt keeps
+    // its CFA in rax and aesni_cbc_encrypt in r11 while calling helpers that preserve them.
+    DWARF_GPRS_CALLER_SAVED =
+        (1 << DWARF_RAX) | (1 << DWARF_RDX) | (1 << DWARF_RCX) | (1 << DWARF_RSI) | (1 << DWARF_RDI) |
+        (1 << DWARF_R8) | (1 << DWARF_R9) | (1 << DWARF_R10) | (1 << DWARF_R11),
+};
+
+static ALWAYS_INLINE void dwarf_unwind_setup_userspace_gprs(struct unwind_gprs* gprs, struct user_regs* regs) {
+    *gprs = regs->gprs;
+}
+
+// Callee-saved registers are restored from the stack by DW_CFA_offset rules, which the unwind table does not
+// carry, so only caller-saved registers survive a DWARF step.
+static ALWAYS_INLINE void dwarf_unwind_gprs_after_dwarf_step(struct unwind_gprs* gprs) {
+    gprs->known &= DWARF_GPRS_CALLER_SAVED;
+}
+
+// Nothing is known about the registers of a frame unwound without CFI.
+static ALWAYS_INLINE void dwarf_unwind_gprs_forget(struct unwind_gprs* gprs) {
+    gprs->known = 0;
+}
+
+static ALWAYS_INLINE bool dwarf_unwind_gprs_get(struct unwind_gprs* gprs, u8 regno, u64* value) {
+    if (regno >= DWARF_GPR_COUNT || !(gprs->known & (1 << regno))) {
+        return false;
+    }
+    *value = gprs->regs[regno];
+    return true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 ALWAYS_INLINE bool dwarf_cfi_eval_ra(
     struct unwind_context* prev,
     struct unwind_context* next,