From 416b65af044ea1f0ac30a91dc16880dca9564e45 Mon Sep 17 00:00:00 2001 From: Martyn Capewell Date: Fri, 7 Oct 2022 10:43:40 +0100 Subject: [PATCH] [gcs] Support Guarded Control Stacks Add support for Guarded Control Stacks in all components, and allow allocating new stacks in the simulator. --- SConstruct | 4 +- src/aarch64/assembler-aarch64.cc | 35 +++ src/aarch64/assembler-aarch64.h | 18 ++ src/aarch64/constants-aarch64.h | 13 +- src/aarch64/cpu-features-auditor-aarch64.cc | 158 ++++++------- src/aarch64/decoder-constants-aarch64.h | 2 +- src/aarch64/decoder-visitor-map-aarch64.h | 1 + src/aarch64/disasm-aarch64.cc | 34 ++- src/aarch64/macro-assembler-aarch64.cc | 16 ++ src/aarch64/macro-assembler-aarch64.h | 31 +++ src/aarch64/simulator-aarch64.cc | 151 +++++++++++- src/aarch64/simulator-aarch64.h | 160 +++++++++++++ src/cpu-features.h | 3 +- test/aarch64/test-api-aarch64.cc | 19 ++ test/aarch64/test-assembler-aarch64.cc | 241 +++++++++++++++++++- test/aarch64/test-disasm-aarch64.cc | 14 ++ tools/code_coverage.log | 3 + 17 files changed, 798 insertions(+), 105 deletions(-) diff --git a/SConstruct b/SConstruct index ac31cb23..b855d646 100644 --- a/SConstruct +++ b/SConstruct @@ -98,7 +98,9 @@ options = { 'CCFLAGS' : ['-O3'], }, 'simulator:aarch64' : { - 'CCFLAGS' : ['-DVIXL_INCLUDE_SIMULATOR_AARCH64'], + 'CCFLAGS' : ['-DVIXL_INCLUDE_SIMULATOR_AARCH64', + '-pthread'], + 'LINKFLAGS' : ['-pthread'] }, 'symbols:on' : { 'CCFLAGS' : ['-g'], diff --git a/src/aarch64/assembler-aarch64.cc b/src/aarch64/assembler-aarch64.cc index c0f66ecd..ad2e7c96 100644 --- a/src/aarch64/assembler-aarch64.cc +++ b/src/aarch64/assembler-aarch64.cc @@ -1918,6 +1918,12 @@ void Assembler::sys(int op, const Register& xt) { } +void Assembler::sysl(int op, const Register& xt) { + VIXL_ASSERT(xt.Is64Bits()); + Emit(SYSL | SysOp(op) | Rt(xt)); +} + + void Assembler::dc(DataCacheOp op, const Register& rt) { if (op == CVAP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCPoP)); if (op == CVADP) VIXL_ASSERT(CPUHas(CPUFeatures::kDCCVADP)); @@ -1930,6 +1936,35 @@ void Assembler::ic(InstructionCacheOp op, const Register& rt) { sys(op, rt); } +void Assembler::gcspushm(const Register& rt) { + VIXL_ASSERT(CPUHas(CPUFeatures::kGCS)); + sys(GCSPUSHM, rt); +} + +void Assembler::gcspopm(const Register& rt) { + VIXL_ASSERT(CPUHas(CPUFeatures::kGCS)); + sysl(GCSPOPM, rt); +} + + +void Assembler::gcsss1(const Register& rt) { + VIXL_ASSERT(CPUHas(CPUFeatures::kGCS)); + sys(GCSSS1, rt); +} + + +void Assembler::gcsss2(const Register& rt) { + VIXL_ASSERT(CPUHas(CPUFeatures::kGCS)); + sysl(GCSSS2, rt); +} + + +void Assembler::chkfeat(const Register& rd) { + VIXL_ASSERT(rd.Is(x16)); + USE(rd); + hint(CHKFEAT); +} + void Assembler::hint(SystemHint code) { hint(static_cast(code)); } diff --git a/src/aarch64/assembler-aarch64.h b/src/aarch64/assembler-aarch64.h index 1028da2c..9bc7076d 100644 --- a/src/aarch64/assembler-aarch64.h +++ b/src/aarch64/assembler-aarch64.h @@ -2158,6 +2158,9 @@ class Assembler : public vixl::internal::AssemblerBase { // System instruction with pre-encoded op (op1:crn:crm:op2). void sys(int op, const Register& xt = xzr); + // System instruction with result. + void sysl(int op, const Register& xt = xzr); + // System data cache operation. void dc(DataCacheOp op, const Register& rt); @@ -7072,6 +7075,21 @@ class Assembler : public vixl::internal::AssemblerBase { // Unsigned Minimum. void umin(const Register& rd, const Register& rn, const Operand& op); + // Check feature status. + void chkfeat(const Register& rd); + + // Guarded Control Stack Push. + void gcspushm(const Register& rt); + + // Guarded Control Stack Pop. + void gcspopm(const Register& rt); + + // Guarded Control Stack Switch Stack 1. + void gcsss1(const Register& rt); + + // Guarded Control Stack Switch Stack 2. + void gcsss2(const Register& rt); + // Emit generic instructions. // Emit raw instructions into the instruction stream. diff --git a/src/aarch64/constants-aarch64.h b/src/aarch64/constants-aarch64.h index 20bd12f9..08469528 100644 --- a/src/aarch64/constants-aarch64.h +++ b/src/aarch64/constants-aarch64.h @@ -389,7 +389,8 @@ enum SystemHint { BTI = 32, BTI_c = 34, BTI_j = 36, - BTI_jc = 38 + BTI_jc = 38, + CHKFEAT = 40 }; enum BranchTargetIdentifier { @@ -534,6 +535,13 @@ enum DataCacheOp { CIGDVAC = CacheOpEncoder<3, 7, 14, 5>::value }; +enum GCSOp { + GCSPUSHM = CacheOpEncoder<3, 7, 7, 0>::value, + GCSPOPM = CacheOpEncoder<3, 7, 7, 1>::value, + GCSSS1 = CacheOpEncoder<3, 7, 7, 2>::value, + GCSSS2 = CacheOpEncoder<3, 7, 7, 3>::value +}; + // Some SVE instructions support a predicate constraint pattern. This is // interpreted as a VL-dependent value, and is typically used to initialise // predicates, or to otherwise limit the number of processed elements. @@ -942,7 +950,8 @@ enum SystemSysOp { SystemSysFixed = 0xD5080000, SystemSysFMask = 0xFFF80000, SystemSysMask = 0xFFF80000, - SYS = SystemSysFixed | 0x00000000 + SYS = SystemSysFixed | 0x00000000, + SYSL = SystemSysFixed | 0x00200000 }; // Exception. diff --git a/src/aarch64/cpu-features-auditor-aarch64.cc b/src/aarch64/cpu-features-auditor-aarch64.cc index 66ac97a5..3925ced3 100644 --- a/src/aarch64/cpu-features-auditor-aarch64.cc +++ b/src/aarch64/cpu-features-auditor-aarch64.cc @@ -1276,91 +1276,93 @@ VIXL_SIMPLE_SVE_VISITOR_LIST(VIXL_DEFINE_SIMPLE_SVE_VISITOR) void CPUFeaturesAuditor::VisitSystem(const Instruction* instr) { RecordInstructionFeaturesScope scope(this); - if (instr->Mask(SystemHintFMask) == SystemHintFixed) { - CPUFeatures required; - switch (instr->GetInstructionBits()) { - case PACIA1716: - case PACIB1716: - case AUTIA1716: - case AUTIB1716: - case PACIAZ: - case PACIASP: - case PACIBZ: - case PACIBSP: - case AUTIAZ: - case AUTIASP: - case AUTIBZ: - case AUTIBSP: - case XPACLRI: - required.Combine(CPUFeatures::kPAuth); - break; - default: - switch (instr->GetImmHint()) { - case ESB: - required.Combine(CPUFeatures::kRAS); - break; - case BTI: - case BTI_j: - case BTI_c: - case BTI_jc: - required.Combine(CPUFeatures::kBTI); - break; - default: - break; - } - break; - } - // These are all HINT instructions, and behave as NOPs if the corresponding - // features are not implemented, so we record the corresponding features - // only if they are available. - if (available_.Has(required)) scope.Record(required); - } else if (instr->Mask(SystemSysMask) == SYS) { - switch (instr->GetSysOp()) { - // DC instruction variants. - case CGVAC: - case CGDVAC: - case CGVAP: - case CGDVAP: - case CIGVAC: - case CIGDVAC: - case GVA: - case GZVA: - scope.Record(CPUFeatures::kMTE); - break; - case CVAP: - scope.Record(CPUFeatures::kDCPoP); - break; - case CVADP: - scope.Record(CPUFeatures::kDCCVADP); - break; - case IVAU: - case CVAC: - case CVAU: - case CIVAC: - case ZVA: - // No special CPU features. - break; - } - } else if (instr->Mask(SystemPStateFMask) == SystemPStateFixed) { - switch (instr->Mask(SystemPStateMask)) { - case CFINV: - scope.Record(CPUFeatures::kFlagM); - break; - case AXFLAG: - case XAFLAG: - scope.Record(CPUFeatures::kAXFlag); - break; - } - } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) { - if (instr->Mask(SystemSysRegMask) == MRS) { + CPUFeatures required; + switch (form_hash_) { + case "pacib1716_hi_hints"_h: + case "pacia1716_hi_hints"_h: + case "pacibsp_hi_hints"_h: + case "paciasp_hi_hints"_h: + case "pacibz_hi_hints"_h: + case "paciaz_hi_hints"_h: + case "autib1716_hi_hints"_h: + case "autia1716_hi_hints"_h: + case "autibsp_hi_hints"_h: + case "autiasp_hi_hints"_h: + case "autibz_hi_hints"_h: + case "autiaz_hi_hints"_h: + case "xpaclri_hi_hints"_h: + required.Combine(CPUFeatures::kPAuth); + break; + case "esb_hi_hints"_h: + required.Combine(CPUFeatures::kRAS); + break; + case "bti_hb_hints"_h: + required.Combine(CPUFeatures::kBTI); + break; + } + + // The instructions above are all HINTs and behave as NOPs if the + // corresponding features are not implemented, so we record the corresponding + // features only if they are available. + if (available_.Has(required)) scope.Record(required); + + switch (form_hash_) { + case "cfinv_m_pstate"_h: + scope.Record(CPUFeatures::kFlagM); + break; + case "axflag_m_pstate"_h: + case "xaflag_m_pstate"_h: + scope.Record(CPUFeatures::kAXFlag); + break; + case "mrs_rs_systemmove"_h: switch (instr->GetImmSystemRegister()) { case RNDR: case RNDRRS: scope.Record(CPUFeatures::kRNG); break; } - } + break; + case "sys_cr_systeminstrs"_h: + switch (instr->GetSysOp()) { + // DC instruction variants. + case CGVAC: + case CGDVAC: + case CGVAP: + case CGDVAP: + case CIGVAC: + case CIGDVAC: + case GVA: + case GZVA: + scope.Record(CPUFeatures::kMTE); + break; + case CVAP: + scope.Record(CPUFeatures::kDCPoP); + break; + case CVADP: + scope.Record(CPUFeatures::kDCCVADP); + break; + case IVAU: + case CVAC: + case CVAU: + case CIVAC: + case ZVA: + // No special CPU features. + break; + case GCSPUSHM: + case GCSSS1: + scope.Record(CPUFeatures::kGCS); + break; + } + break; + case "sysl_rc_systeminstrs"_h: + switch (instr->GetSysOp()) { + case GCSPOPM: + case GCSSS2: + scope.Record(CPUFeatures::kGCS); + break; + } + break; } } diff --git a/src/aarch64/decoder-constants-aarch64.h b/src/aarch64/decoder-constants-aarch64.h index 70e01a10..af50a552 100644 --- a/src/aarch64/decoder-constants-aarch64.h +++ b/src/aarch64/decoder-constants-aarch64.h @@ -3764,7 +3764,7 @@ static const DecodeMapping kDecodeMapping[] = { {"001110"_b, "autiaz_hi_hints"}, {"001111"_b, "autibz_hi_hints"}, {"0100xx"_b, "bti_hb_hints"}, - {"010100"_b, "chkfeat_hi_hints"}, + {"010100"_b, "chkfeat_hf_hints"}, {"0101x1"_b, "hint_hm_hints"}, {"01x110"_b, "hint_hm_hints"}, {"10xxxx"_b, "hint_hm_hints"}, diff --git a/src/aarch64/decoder-visitor-map-aarch64.h b/src/aarch64/decoder-visitor-map-aarch64.h index b40e0ae0..a0a6ef22 100644 --- a/src/aarch64/decoder-visitor-map-aarch64.h +++ b/src/aarch64/decoder-visitor-map-aarch64.h @@ -2591,6 +2591,7 @@ {"dmb_bo_barriers"_h, &VISITORCLASS::VisitSystem}, \ {"dsb_bo_barriers"_h, &VISITORCLASS::VisitSystem}, \ {"hint_hm_hints"_h, &VISITORCLASS::VisitSystem}, \ + {"chkfeat_hf_hints"_h, &VISITORCLASS::VisitSystem}, \ {"mrs_rs_systemmove"_h, &VISITORCLASS::VisitSystem}, \ {"msr_sr_systemmove"_h, &VISITORCLASS::VisitSystem}, \ {"psb_hc_hints"_h, &VISITORCLASS::VisitSystem}, \ diff --git a/src/aarch64/disasm-aarch64.cc b/src/aarch64/disasm-aarch64.cc index 6a25271f..9f53e813 100644 --- a/src/aarch64/disasm-aarch64.cc +++ b/src/aarch64/disasm-aarch64.cc @@ -2018,7 +2018,7 @@ void Disassembler::DisassembleNoArgs(const Instruction *instr) { void Disassembler::VisitSystem(const Instruction *instr) { const char *mnemonic = mnemonic_.c_str(); - const char *form = "(System)"; + const char *form = ""; const char *suffix = NULL; switch (form_hash_) { @@ -2047,6 +2047,10 @@ void Disassembler::VisitSystem(const Instruction *instr) { break; } break; + case "chkfeat_hf_hints"_h: + mnemonic = "chkfeat"; + form = "x16"; + break; case "hint_hm_hints"_h: form = "'IH"; break; @@ -2067,9 +2071,6 @@ void Disassembler::VisitSystem(const Instruction *instr) { break; } case Hash("sys_cr_systeminstrs"): { - mnemonic = "dc"; - suffix = ", 'Xt"; - const std::map dcop = { {IVAU, "ivau"}, {CVAC, "cvac"}, @@ -2092,17 +2093,36 @@ void Disassembler::VisitSystem(const Instruction *instr) { if (dcop.count(sysop)) { if (sysop == IVAU) { mnemonic = "ic"; + } else { + mnemonic = "dc"; } form = dcop.at(sysop); + suffix = ", 'Xt"; + } else if (sysop == GCSSS1) { + mnemonic = "gcsss1"; + form = "'Xt"; + } else if (sysop == GCSPUSHM) { + mnemonic = "gcspushm"; + form = "'Xt"; } else { mnemonic = "sys"; form = "'G1, 'Kn, 'Km, 'G2"; - if (instr->GetRt() == 31) { - suffix = NULL; + if (instr->GetRt() < 31) { + suffix = ", 'Xt"; } - break; } + break; } + case "sysl_rc_systeminstrs"_h: + uint32_t sysop = instr->GetSysOp(); + if (sysop == GCSPOPM) { + mnemonic = "gcspopm"; + form = (instr->GetRt() == 31) ? "" : "'Xt"; + } else if (sysop == GCSSS2) { + mnemonic = "gcsss2"; + form = "'Xt"; + } + break; } Format(instr, mnemonic, form, suffix); } diff --git a/src/aarch64/macro-assembler-aarch64.cc b/src/aarch64/macro-assembler-aarch64.cc index 8ee7144f..af90a423 100644 --- a/src/aarch64/macro-assembler-aarch64.cc +++ b/src/aarch64/macro-assembler-aarch64.cc @@ -1970,6 +1970,22 @@ void MacroAssembler::Setf16(const Register& wn) { setf16(wn); } +void MacroAssembler::Chkfeat(const Register& xdn) { + VIXL_ASSERT(allow_macro_instructions_); + MacroEmissionCheckScope guard(this); + if (xdn.Is(x16)) { + chkfeat(xdn); + } else { + UseScratchRegisterScope temps(this); + if (temps.TryAcquire(x16)) { + Mov(x16, xdn); + chkfeat(x16); + Mov(xdn, x16); + } else { + VIXL_ABORT(); + } + } +} #define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \ void MacroAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \ diff --git a/src/aarch64/macro-assembler-aarch64.h b/src/aarch64/macro-assembler-aarch64.h index a46e2ccc..f6fc4d74 100644 --- a/src/aarch64/macro-assembler-aarch64.h +++ b/src/aarch64/macro-assembler-aarch64.h @@ -2717,6 +2717,27 @@ class MacroAssembler : public Assembler, public MacroAssemblerInterface { subps(xd, xn, xm); } void Cmpp(const Register& xn, const Register& xm) { Subps(xzr, xn, xm); } + void Chkfeat(const Register& xdn); + void Gcspushm(const Register& rt) { + VIXL_ASSERT(allow_macro_instructions_); + SingleEmissionCheckScope guard(this); + gcspushm(rt); + } + void Gcspopm(const Register& rt = xzr) { + VIXL_ASSERT(allow_macro_instructions_); + SingleEmissionCheckScope guard(this); + gcspopm(rt); + } + void Gcsss1(const Register& rt) { + VIXL_ASSERT(allow_macro_instructions_); + SingleEmissionCheckScope guard(this); + gcsss1(rt); + } + void Gcsss2(const Register& rt) { + VIXL_ASSERT(allow_macro_instructions_); + SingleEmissionCheckScope guard(this); + gcsss2(rt); + } // NEON 3 vector register instructions. #define NEON_3VREG_MACRO_LIST(V) \ @@ -8584,6 +8605,16 @@ class UseScratchRegisterScope { return AcquireFrom(available, kGoverningPRegisterMask).P(); } + // TODO: extend to other scratch register lists. + bool TryAcquire(const Register& required_reg) { + CPURegList* list = masm_->GetScratchRegisterList(); + if (list->IncludesAliasOf(required_reg)) { + list->Remove(required_reg); + return true; + } + return false; + } + Register AcquireRegisterOfSize(int size_in_bits); Register AcquireSameSizeAs(const Register& reg) { return AcquireRegisterOfSize(reg.GetSizeInBits()); diff --git a/src/aarch64/simulator-aarch64.cc b/src/aarch64/simulator-aarch64.cc index b9a1bdcc..81bab07d 100644 --- a/src/aarch64/simulator-aarch64.cc +++ b/src/aarch64/simulator-aarch64.cc @@ -540,7 +540,9 @@ asm(R"( Simulator::Simulator(Decoder* decoder, FILE* stream, SimStack::Allocated stack) : memory_(std::move(stack)), last_instr_(NULL), - cpu_features_auditor_(decoder, CPUFeatures::All()) { + cpu_features_auditor_(decoder, CPUFeatures::All()), + gcs_(kGCSNoStack), + gcs_enabled_(false) { // Ensure that shift operations act as the simulator expects. VIXL_ASSERT((static_cast(-1) >> 1) == -1); VIXL_ASSERT((static_cast(-1) >> 1) == 0x7fffffff); @@ -660,6 +662,8 @@ void Simulator::ResetState() { ResetPRegisters(); WriteSp(memory_.GetStack().GetBase()); + ResetGCSState(); + EnableGCSCheck(); pc_ = NULL; pc_modified_ = false; @@ -697,6 +701,9 @@ Simulator::~Simulator() { delete print_disasm_; close(placeholder_pipe_fd_[0]); close(placeholder_pipe_fd_[1]); + if (IsAllocatedGCS(gcs_)) { + GetGCSManager().FreeStack(gcs_); + } } @@ -1797,6 +1804,18 @@ void Simulator::PrintSystemRegister(SystemRegister id) { } } +void Simulator::PrintGCS(bool is_push, uint64_t addr, size_t entry) { + const char* arrow = is_push ? "<-" : "->"; + fprintf(stream_, + "# %sgcs0x%04" PRIx64 "[%" PRIu64 "]: %s %s 0x%016" PRIxPTR "\n", + clr_flag_name, + gcs_, + entry, + clr_normal, + arrow, + addr); +} + uint16_t Simulator::PrintPartialAccess(uint16_t access_mask, uint16_t future_access_mask, int struct_element_count, @@ -3774,6 +3793,7 @@ void Simulator::VisitUnconditionalBranch(const Instruction* instr) { switch (instr->Mask(UnconditionalBranchMask)) { case BL: WriteLr(instr->GetNextInstruction()); + GCSPush(reinterpret_cast(instr->GetNextInstruction())); VIXL_FALLTHROUGH(); case B: WritePc(instr->GetImmPCOffsetTarget()); @@ -3817,6 +3837,7 @@ void Simulator::VisitUnconditionalBranchToRegister(const Instruction* instr) { bool authenticate = false; bool link = false; bool ret = false; + bool compare_gcs = false; uint64_t addr = ReadXRegister(instr->GetRn()); uint64_t context = 0; @@ -3853,16 +3874,13 @@ void Simulator::VisitUnconditionalBranchToRegister(const Instruction* instr) { context = ReadXRegister(31, Reg31IsStackPointer); VIXL_FALLTHROUGH(); case RET: + compare_gcs = true; ret = true; break; default: VIXL_UNREACHABLE(); } - if (link) { - WriteLr(instr->GetNextInstruction()); - } - if (authenticate) { PACKey key = (instr->ExtractBit(10) == 0) ? kPACKeyIA : kPACKeyIB; addr = AuthPAC(addr, context, key, kInstructionPointer); @@ -3873,6 +3891,33 @@ void Simulator::VisitUnconditionalBranchToRegister(const Instruction* instr) { } } + if (compare_gcs) { + uint64_t expected_lr = GCSPeek(); + char msg[128]; + if (expected_lr != 0) { + if ((expected_lr & 0x3) != 0) { + snprintf(msg, + sizeof(msg), + "GCS contains misaligned return address: 0x%016lx\n", + expected_lr); + ReportGCSFailure(msg); + } else if ((addr != 0) && (addr != expected_lr)) { + snprintf(msg, + sizeof(msg), + "GCS mismatch: lr = 0x%016lx, gcs = 0x%016lx\n", + addr, + expected_lr); + ReportGCSFailure(msg); + } + GCSPop(); + } + } + + if (link) { + WriteLr(instr->GetNextInstruction()); + GCSPush(reinterpret_cast(instr->GetNextInstruction())); + } + if (!ret) { // Check for interceptions to the target address, if one is found, call it. MetaDataDepot::BranchInterceptionAbstract* interception = @@ -6909,6 +6954,14 @@ void Simulator::VisitSystem(const Instruction* instr) { VIXL_UNIMPLEMENTED(); } break; + case "chkfeat_hf_hints"_h: { + uint64_t feat_select = ReadXRegister(16); + uint64_t gcs_enabled = IsGCSCheckEnabled() ? 1 : 0; + feat_select &= ~gcs_enabled; + WriteXRegister(16, feat_select); + break; + } + case "hint_hm_hints"_h: case "nop_hi_hints"_h: case "esb_hi_hints"_h: case "csdb_hi_hints"_h: @@ -6992,9 +7045,64 @@ void Simulator::VisitSystem(const Instruction* instr) { case "isb_bi_barriers"_h: __sync_synchronize(); break; - case "sys_cr_systeminstrs"_h: - SysOp_W(instr->GetSysOp(), ReadXRegister(instr->GetRt())); + case "sys_cr_systeminstrs"_h: { + uint64_t rt = ReadXRegister(instr->GetRt()); + uint32_t sysop = instr->GetSysOp(); + if (sysop == GCSSS1) { + uint64_t incoming_size = rt >> 32; + // Drop upper 32 bits to get GCS index. + uint64_t incoming_gcs = rt & 0xffffffff; + uint64_t outgoing_gcs = ActivateGCS(incoming_gcs); + uint64_t incoming_seal = GCSPop(); + if (((incoming_seal ^ rt) != 1) || + (GetActiveGCSPtr()->size() != incoming_size)) { + char msg[128]; + snprintf(msg, + sizeof(msg), + "GCS: invalid incoming stack: 0x%016" PRIx64 "\n", + incoming_seal); + ReportGCSFailure(msg); + } + GCSPush(outgoing_gcs + 5); + } else if (sysop == GCSPUSHM) { + GCSPush(ReadXRegister(instr->GetRt())); + } else { + SysOp_W(sysop, rt); + } + break; + } + case "sysl_rc_systeminstrs"_h: { + uint32_t sysop = instr->GetSysOp(); + if (sysop == GCSPOPM) { + uint64_t addr = GCSPop(); + WriteXRegister(instr->GetRt(), addr); + } else if (sysop == GCSSS2) { + uint64_t outgoing_gcs = GCSPop(); + // Check for token inserted by gcsss1. + if ((outgoing_gcs & 7) != 5) { + char msg[128]; + snprintf(msg, + sizeof(msg), + "GCS: outgoing stack has no token: 0x%016" PRIx64 "\n", + outgoing_gcs); + ReportGCSFailure(msg); + } + uint64_t incoming_gcs = ActivateGCS(outgoing_gcs); + outgoing_gcs &= ~UINT64_C(0x3ff); + + // Encode the size into the outgoing stack seal, to check later. + uint64_t size = GetActiveGCSPtr()->size(); + VIXL_ASSERT(IsUint32(size)); + VIXL_ASSERT(IsUint32(outgoing_gcs + 1)); + uint64_t outgoing_seal = (size << 32) | (outgoing_gcs + 1); + GCSPush(outgoing_seal); + ActivateGCS(incoming_gcs); + WriteXRegister(instr->GetRt(), outgoing_seal - 1); + } else { + VIXL_UNIMPLEMENTED(); + } break; + } default: VIXL_UNIMPLEMENTED(); } @@ -14796,12 +14904,35 @@ void Simulator::DoRuntimeCall(const Instruction* instr) { reinterpret_cast(call_wrapper_address); if (static_cast(call_type) == kCallRuntime) { - WriteRegister(kLinkRegCode, - instr->GetInstructionAtOffset(kRuntimeCallLength)); + const Instruction* addr = instr->GetInstructionAtOffset(kRuntimeCallLength); + WriteLr(addr); + GCSPush(reinterpret_cast(addr)); } runtime_call_wrapper(this, function_address); // Read the return address from `lr` and write it into `pc`. - WritePc(ReadRegister(kLinkRegCode)); + uint64_t addr = ReadRegister(kLinkRegCode); + if (IsGCSCheckEnabled()) { + uint64_t expected_lr = GCSPeek(); + char msg[128]; + if (expected_lr != 0) { + if ((expected_lr & 0x3) != 0) { + snprintf(msg, + sizeof(msg), + "GCS contains misaligned return address: 0x%016lx\n", + expected_lr); + ReportGCSFailure(msg); + } else if ((addr != 0) && (addr != expected_lr)) { + snprintf(msg, + sizeof(msg), + "GCS mismatch: lr = 0x%016lx, gcs = 0x%016lx\n", + addr, + expected_lr); + ReportGCSFailure(msg); + } + GCSPop(); + } + } + WritePc(reinterpret_cast(addr)); } #else void Simulator::DoRuntimeCall(const Instruction* instr) { diff --git a/src/aarch64/simulator-aarch64.h b/src/aarch64/simulator-aarch64.h index b4b7aa62..760fa6cb 100644 --- a/src/aarch64/simulator-aarch64.h +++ b/src/aarch64/simulator-aarch64.h @@ -28,6 +28,7 @@ #define VIXL_AARCH64_SIMULATOR_AARCH64_H_ #include +#include #include #include @@ -2529,12 +2530,16 @@ class Simulator : public DecoderVisitor { // Other state updates, including system registers. void PrintSystemRegister(SystemRegister id); void PrintTakenBranch(const Instruction* target); + void PrintGCS(bool is_push, uint64_t addr, size_t entry); void LogSystemRegister(SystemRegister id) { if (ShouldTraceSysRegs()) PrintSystemRegister(id); } void LogTakenBranch(const Instruction* target) { if (ShouldTraceBranches()) PrintTakenBranch(target); } + void LogGCS(bool is_push, uint64_t addr, size_t entry) { + if (ShouldTraceSysRegs()) PrintGCS(is_push, addr, entry); + } // Trace memory accesses. @@ -5352,6 +5357,161 @@ class Simulator : public DecoderVisitor { // Debugger for the simulator. std::unique_ptr debugger_; + + // The Guarded Control Stack is represented using a vector, where the more + // recently stored addresses are at higher-numbered indices. + using GuardedControlStack = std::vector; + + // The GCSManager handles the synchronisation of GCS across multiple + // Simulator instances. Each Simulator has its own stack, but all share + // a GCSManager instance. This allows exchanging stacks between Simulators + // in a threaded application. + class GCSManager { + public: + // Allocate a new Guarded Control Stack and add it to the vector of stacks. + uint64_t AllocateStack() { + const std::lock_guard lock(stacks_mtx_); + + GuardedControlStack* new_stack = new GuardedControlStack; + uint64_t result; + + // Put the new stack into the first available slot. + for (result = 0; result < stacks_.size(); result++) { + if (stacks_[result] == nullptr) { + stacks_[result] = new_stack; + break; + } + } + + // If there were no slots, create a new one. + if (result == stacks_.size()) { + stacks_.push_back(new_stack); + } + + // Shift the index to look like a stack pointer aligned to a page. + result <<= kPageSizeLog2; + + // Push the tagged index onto the new stack as a seal. + new_stack->push_back(result + 1); + return result; + } + + // Free a Guarded Control Stack and set the stacks_ slot to null. + void FreeStack(uint64_t gcs) { + const std::lock_guard lock(stacks_mtx_); + uint64_t gcs_index = GetGCSIndex(gcs); + GuardedControlStack* gcsptr = stacks_[gcs_index]; + if (gcsptr == nullptr) { + VIXL_ABORT_WITH_MSG("Tried to free unallocated GCS "); + } else { + delete gcsptr; + stacks_[gcs_index] = nullptr; + } + } + + // Get a pointer to the GCS vector using a GCS id. + GuardedControlStack* GetGCSPtr(uint64_t gcs) const { + return stacks_[GetGCSIndex(gcs)]; + } + + private: + uint64_t GetGCSIndex(uint64_t gcs) const { return gcs >> 12; } + + std::vector stacks_; + std::mutex stacks_mtx_; + }; + + // A GCS id indicating no GCS has been allocated. + static const uint64_t kGCSNoStack = kPageSize - 1; + uint64_t gcs_; + bool gcs_enabled_; + + public: + GCSManager& GetGCSManager() { + static GCSManager manager; + return manager; + } + + void EnableGCSCheck() { gcs_enabled_ = true; } + void DisableGCSCheck() { gcs_enabled_ = false; } + bool IsGCSCheckEnabled() const { return gcs_enabled_; } + + private: + bool IsAllocatedGCS(uint64_t gcs) const { return gcs != kGCSNoStack; } + void ResetGCSState() { + GCSManager& m = GetGCSManager(); + if (IsAllocatedGCS(gcs_)) { + m.FreeStack(gcs_); + } + ActivateGCS(m.AllocateStack()); + GCSPop(); // Remove seal. + } + + GuardedControlStack* GetGCSPtr(uint64_t gcs) { + GCSManager& m = GetGCSManager(); + GuardedControlStack* result = m.GetGCSPtr(gcs); + return result; + } + GuardedControlStack* GetActiveGCSPtr() { return GetGCSPtr(gcs_); } + + uint64_t ActivateGCS(uint64_t gcs) { + uint64_t outgoing_gcs = gcs_; + gcs_ = gcs; + return outgoing_gcs; + } + + void GCSPush(uint64_t addr) { + GetActiveGCSPtr()->push_back(addr); + size_t entry = GetActiveGCSPtr()->size() - 1; + LogGCS(/* is_push = */ true, addr, entry); + } + + uint64_t GCSPop() { + GuardedControlStack* gcs = GetActiveGCSPtr(); + if (gcs->empty()) { + return 0; + } + uint64_t return_addr = gcs->back(); + size_t entry = gcs->size() - 1; + gcs->pop_back(); + LogGCS(/* is_push = */ false, return_addr, entry); + return return_addr; + } + + uint64_t GCSPeek() { + GuardedControlStack* gcs = GetActiveGCSPtr(); + if (gcs->empty()) { + return 0; + } + uint64_t return_addr = gcs->back(); + return return_addr; + } + + void ReportGCSFailure(const char* msg) { + if (IsGCSCheckEnabled()) { + GuardedControlStack* gcs = GetActiveGCSPtr(); + printf("%s", msg); + if (gcs == nullptr) { + printf("GCS pointer is null\n"); + } else { + printf("GCS records, most recent first:\n"); + int most_recent_index = static_cast(gcs->size()) - 1; + for (int i = 0; i < 8; i++) { + if (!gcs->empty()) { + uint64_t entry = gcs->back(); + gcs->pop_back(); + int index = most_recent_index - i; + printf(" gcs%" PRIu64 "[%d]: 0x%016" PRIx64 "\n", + gcs_, + index, + entry); + } + } + printf("End of GCS records.\n"); + } + VIXL_ABORT_WITH_MSG("GCS failed "); + } + } }; #if defined(VIXL_HAS_SIMULATED_RUNTIME_CALL_SUPPORT) && __cplusplus < 201402L diff --git a/src/cpu-features.h b/src/cpu-features.h index 97eb661a..1a041f66 100644 --- a/src/cpu-features.h +++ b/src/cpu-features.h @@ -201,7 +201,8 @@ namespace vixl { /* Extended BFloat16 instructions */ \ V(kEBF16, "EBF16", "ebf16") \ V(kSVE_EBF16, "EBF16 (SVE)", "sveebf16") \ - V(kCSSC, "CSSC", "cssc") + V(kCSSC, "CSSC", "cssc") \ + V(kGCS, "GCS", "gcs") // clang-format on diff --git a/test/aarch64/test-api-aarch64.cc b/test/aarch64/test-api-aarch64.cc index c724f178..3ac9efb7 100644 --- a/test/aarch64/test-api-aarch64.cc +++ b/test/aarch64/test-api-aarch64.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include "test-runner.h" #include "test-utils.h" @@ -1763,6 +1764,24 @@ TEST(sim_stack) { VIXL_CHECK(s.IsAccessInGuardRegion(s.GetLimit() - 1280, 2048)); VIXL_CHECK(s.IsAccessInGuardRegion(s.GetLimit() - 1280, 10000)); } + +void AllocateAndFreeGCS() { + Decoder d; + Simulator s(&d); + + for (int i = 0; i < 100000; i++) { + uint64_t gcs = s.GetGCSManager().AllocateStack(); + s.GetGCSManager().FreeStack(gcs); + } +} + +TEST(sim_gcs_manager) { + std::thread t1(AllocateAndFreeGCS); + std::thread t2(AllocateAndFreeGCS); + + t1.join(); + t2.join(); +} #endif } // namespace aarch64 diff --git a/test/aarch64/test-assembler-aarch64.cc b/test/aarch64/test-assembler-aarch64.cc index 00155471..81d6b7a5 100644 --- a/test/aarch64/test-assembler-aarch64.cc +++ b/test/aarch64/test-assembler-aarch64.cc @@ -7501,11 +7501,12 @@ static void BtiHelper(Register ipreg) { __ Blr(x0); __ Adr(ipreg, &jump_call_target); __ Blr(ipreg); - __ Adr(lr, &done); // Make Ret return to done label. + __ Mov(lr, 0); // Zero lr so we branch to done. __ Br(ipreg); __ Bind(&call_target, EmitBTI_c); __ Ret(); __ Bind(&jump_call_target, EmitBTI_jc); + __ Cbz(lr, &done); __ Ret(); __ Bind(&done); END(); @@ -7529,28 +7530,36 @@ TEST(unguarded_bti_is_nop) { SETUP_WITH_FEATURES(CPUFeatures::kBTI); Label start, none, c, j, jc; + Label jump_to_c, call_to_j; START(); __ B(&start); __ Bind(&none, EmitBTI); __ Bind(&c, EmitBTI_c); __ Bind(&j, EmitBTI_j); __ Bind(&jc, EmitBTI_jc); - VIXL_CHECK(__ GetSizeOfCodeGeneratedSince(&none) == 4 * kInstructionSize); + __ Hint(BTI); + __ Hint(BTI_c); + __ Hint(BTI_j); + __ Hint(BTI_jc); + VIXL_CHECK(__ GetSizeOfCodeGeneratedSince(&none) == 8 * kInstructionSize); + __ Cmp(x1, 1); + __ B(lt, &jump_to_c); + __ B(eq, &call_to_j); __ Ret(); - Label jump_to_c, call_to_j; __ Bind(&start); __ Adr(x0, &none); - __ Adr(lr, &jump_to_c); + __ Mov(x1, 0); __ Br(x0); __ Bind(&jump_to_c); __ Adr(x0, &c); - __ Adr(lr, &call_to_j); + __ Mov(x1, 1); __ Br(x0); __ Bind(&call_to_j); __ Adr(x0, &j); + __ Mov(x1, 2); __ Blr(x0); END(); @@ -14723,6 +14732,228 @@ TEST(cssc_smax) { MinMaxHelper(op, true, s64min, s64max, 0, s64max); } +TEST(gcs_chkfeat) { + SETUP(); + + START(); + __ Mov(x16, 0x0123'4567'89ab'cdef); + __ Chkfeat(x16); + __ Mov(x0, x16); + + __ Mov(x1, 0x0123'4567'89ab'cdef); + __ Chkfeat(x1); + END(); + + if (CAN_RUN()) { + RUN(); + ASSERT_EQUAL_64(0x0123'4567'89ab'cdee, x0); + ASSERT_EQUAL_64(x0, x1); + } +} + +TEST(gcs_feature_off) { + SETUP(); + + START(); +#ifdef VIXL_INCLUDE_SIMULATOR_AARCH64 + simulator.DisableGCSCheck(); +#else +// TODO: Disable GCS via operating system for this test, here and in the +// gcs_off_pac_on test below. +#endif + __ Mov(x16, 0x0123'4567'89ab'cdef); + __ Chkfeat(x16); + + // This sequence would fail with GCS enabled. + Label lab, end; + __ Bl(&lab); + __ B(&end); + + __ Bind(&lab); + __ Adr(lr, &end); + __ Ret(); + + __ Bind(&end); + END(); + + if (CAN_RUN()) { + // TODO: This will currently fail on GCS-supporting hardware. + RUN(); + ASSERT_EQUAL_64(0x0123'4567'89ab'cdef, x16); + } +} + +TEST(gcs_gcspushm) { + SETUP_WITH_FEATURES(CPUFeatures::kGCS); + + Label ret; + START(); + __ Adr(x0, &ret); + __ Gcspushm(x0); + __ Ret(x0); + __ Nop(); + __ Bind(&ret); + END(); + + if (CAN_RUN()) { + RUN(); + } +} + +TEST(gcs_gcspopm) { + SETUP_WITH_FEATURES(CPUFeatures::kGCS); + + Label lab, ret; + START(); + __ Adr(x0, &ret); + __ Bl(&lab); + __ Bind(&ret); + __ Nop(); + __ Bind(&lab); + __ Gcspopm(x1); + END(); + + if (CAN_RUN()) { + RUN(); + ASSERT_EQUAL_64(x0, x1); + } +} + +TEST(gcs_gcsss1) { + SETUP_WITH_FEATURES(CPUFeatures::kGCS); + + START(); +#ifdef VIXL_INCLUDE_SIMULATOR_AARCH64 + uint64_t new_gcs = simulator.GetGCSManager().AllocateStack(); + __ Mov(x0, new_gcs); +#else +// TODO: Request new GCS from the operating system. +#endif + + // Partial stack swap to check GCS has changed, and a token is at the top + // of the new stack. + __ Gcsss1(x0); + __ Gcspopm(x1); + + __ Bic(x0, x0, 7); // Clear LSB of new GCS. + __ Bic(x2, x1, 7); // Clear LSB of old GCS. + __ Cmp(x0, x2); + __ Cset(x0, eq); + __ And(x1, x1, 7); // In progress token. + END(); + + if (CAN_RUN()) { + RUN(); + ASSERT_EQUAL_64(0, x0); // GCS must not be equal. + ASSERT_EQUAL_64(5, x1); // In progress token must be present. + } +} + +// TODO: Add extra tests for combinations of PAC and GCS enabled. +TEST(gcs_stack_swap) { + SETUP_WITH_FEATURES(CPUFeatures::kGCS); + + START(); + Label stack_swap, sub_fn, end; +#ifdef VIXL_INCLUDE_SIMULATOR_AARCH64 + uint64_t new_gcs = simulator.GetGCSManager().AllocateStack(); + __ Mov(x0, new_gcs); +#else +// TODO: Request new GCS from the operating system. +#endif + __ Bl(&stack_swap); + __ B(&end); + + __ Bind(&stack_swap); + __ Gcsss1(x0); // x0 = new GCS. + __ Gcsss2(x1); // x1 = old GCS. + __ Mov(x29, lr); + __ Bl(&sub_fn); + __ Mov(lr, x29); + __ Gcsss1(x1); // Restore old GCS. + __ Gcsss2(x0); + __ Ret(); + + __ Bind(&sub_fn); + __ Mov(x2, 42); + __ Ret(); + + __ Bind(&end); + END(); + + if (CAN_RUN()) { + RUN(); + ASSERT_EQUAL_64(42, x2); + } +} + +TEST(gcs_off_pac_on) { + SETUP_WITH_FEATURES(CPUFeatures::kPAuth); + + START(); +#ifdef VIXL_INCLUDE_SIMULATOR_AARCH64 + simulator.DisableGCSCheck(); +#else +// TODO: Disable GCS via operating system for this test, and enable for native. +#endif + __ Mov(x16, 1); + __ Chkfeat(x16); + __ Mov(x1, x16); + + Label fn1, after_fn1; + + __ Mov(x28, sp); + __ Mov(x29, lr); + __ Mov(sp, 0x477d469dec0b8760); + + __ Mov(x0, 0); + __ B(&after_fn1); + + __ Bind(&fn1); + __ Mov(x0, 42); + __ Paciasp(); + __ Retaa(); + + __ Bind(&after_fn1); + __ Bl(&fn1); + + __ Mov(sp, x28); + __ Mov(lr, x29); + END(); + + if (CAN_RUN()) { + RUN(); + + ASSERT_EQUAL_64(42, x0); + ASSERT_EQUAL_64(1, x1); + } +} + +#ifdef VIXL_NEGATIVE_TESTING +TEST(gcs_negative_test) { + SETUP_WITH_FEATURES(CPUFeatures::kGCS); + + Label fn, bad_return_addr, done; + START(); + __ Bl(&fn); + __ Nop(); // GCS enforces that fn() returns here... + + __ Bind(&bad_return_addr); + __ B(&done); // ... but this test attempts to return here. + + __ Bind(&fn); + __ Adr(lr, &bad_return_addr); + __ Ret(); + + __ Bind(&done); + END(); + + if (CAN_RUN()) { + MUST_FAIL_WITH_MESSAGE(RUN(), "GCS failed"); + } +} +#endif // VIXL_NEGATIVE_TESTING + #ifdef VIXL_INCLUDE_SIMULATOR_AARCH64 // Test the pseudo-instructions that control CPUFeatures dynamically in the // Simulator. These are used by the test infrastructure itself, but in a fairly diff --git a/test/aarch64/test-disasm-aarch64.cc b/test/aarch64/test-disasm-aarch64.cc index 28eb2836..7c8f2ccf 100644 --- a/test/aarch64/test-disasm-aarch64.cc +++ b/test/aarch64/test-disasm-aarch64.cc @@ -3359,6 +3359,20 @@ TEST(cssc) { CLEANUP(); } +TEST(gcs) { + SETUP(); + + COMPARE_MACRO(Chkfeat(x16), "chkfeat x16"); + COMPARE_MACRO(Gcspopm(x0), "gcspopm x0"); + COMPARE_MACRO(Gcspopm(), "gcspopm"); + COMPARE_MACRO(Gcspopm(xzr), "gcspopm"); + COMPARE_MACRO(Gcsss1(x4), "gcsss1 x4"); + COMPARE_MACRO(Gcsss2(x2), "gcsss2 x2"); + COMPARE_MACRO(Gcspushm(x1), "gcspushm x1"); + + CLEANUP(); +} + TEST(architecture_features) { SETUP(); diff --git a/tools/code_coverage.log b/tools/code_coverage.log index 091124c8..f913151b 100644 --- a/tools/code_coverage.log +++ b/tools/code_coverage.log @@ -14,12 +14,15 @@ 1660224011 82.79% 97.51% 95.50% 1663161852 82.79% 97.51% 95.50% 1666104118 82.79% 97.51% 95.50% +1668785529 82.75% 97.44% 95.40% 1669202345 82.79% 97.51% 95.51% 1673432155 82.79% 97.51% 95.51% 1677171445 82.78% 97.56% 94.81% 1681814646 82.90% 97.57% 94.87% 1686666000 82.90% 97.57% 94.87% 1693487542 82.91% 97.57% 94.87% +1694008240 82.72% 97.50% 94.95% +1697036303 82.87% 97.56% 94.76% 1702052331 82.89% 97.59% 94.77% 1706691191 82.87% 97.59% 94.74% 1707395574 82.89% 97.59% 94.77%