diff --git a/lldb/source/Plugins/Architecture/AArch64/ArchitectureAArch64.cpp b/lldb/source/Plugins/Architecture/AArch64/ArchitectureAArch64.cpp index 6a072354972ac..1934768876b5d 100644 --- a/lldb/source/Plugins/Architecture/AArch64/ArchitectureAArch64.cpp +++ b/lldb/source/Plugins/Architecture/AArch64/ArchitectureAArch64.cpp @@ -123,6 +123,11 @@ bool ArchitectureAArch64::ReconfigureRegisterInfo(DynamicRegisterInfo ®_info, if (!vg_reg_value && !svg_reg_value) return false; + if (!vg_reg_value) { + // This must be an SME only system, so VG == SVG. + vg_reg_value = svg_reg_value; + } + auto regs = reg_info.registers(); if (vg_reg_value) UpdateARM64SVERegistersInfos(regs, *vg_reg_value); diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp index 294a446686f22..c81b907e4ac1a 100644 --- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp +++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp @@ -108,19 +108,18 @@ NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux( if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, native_thread.GetID(), ®set, &ioVec, sizeof(sve_header)) - .Success()) { + .Success()) opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSVE); - // We may also have the Scalable Matrix Extension (SME) which adds a - // streaming SVE mode. - ioVec.iov_len = sizeof(sve_header); - regset = NT_ARM_SSVE; - if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, - native_thread.GetID(), ®set, - &ioVec, sizeof(sve_header)) - .Success()) - opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE); - } + // We may have the Scalable Matrix Extension (SME) which adds a + // streaming SVE mode. Systems can have SVE and/or SME. + ioVec.iov_len = sizeof(sve_header); + regset = NT_ARM_SSVE; + if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, + native_thread.GetID(), ®set, + &ioVec, sizeof(sve_header)) + .Success()) + opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE); sve::user_za_header za_header; ioVec.iov_base = &za_header; @@ -291,13 +290,17 @@ NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info, src = (uint8_t *)GetGPRBuffer() + offset; } else if (IsFPR(reg)) { - if (m_sve_state == SVEState::Disabled) { - // SVE is disabled take legacy route for FPU register access + if (m_sve_state == SVEState::Disabled || + m_sve_state == SVEState::StreamingFPSIMD) { + // FP registers come from the FP register set when: + // * We only have SVE in streaming mode, and we are in non-streaming mode. + // * We only have SIMD, no SVE in any mode. error = ReadFPR(); if (error.Fail()) return error; - offset = CalculateFprOffset(reg_info); + offset = CalculateFprOffset(reg_info, + m_sve_state == SVEState::StreamingFPSIMD); assert(offset < GetFPRSize()); src = (uint8_t *)GetFPRBuffer() + offset; } else { @@ -349,8 +352,54 @@ NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info, return Status::FromErrorString("SVE disabled or not supported"); if (GetRegisterInfo().IsSVERegVG(reg)) { + error = ReadSVEHeader(); + if (error.Fail()) + return error; + sve_vg = GetSVERegVG(); src = (uint8_t *)&sve_vg; + } else if (m_sve_state == SVEState::StreamingFPSIMD) { + // When we only have streaming SVE and we are in non-streaming mode, + // we cannot read streaming SVE registers. + + // P and FFR show as 0s. + if (GetRegisterInfo().IsSVEPReg(reg) || + GetRegisterInfo().IsSVERegFFR(reg)) { + std::vector fake_reg(reg_info->byte_size, 0); + reg_value.SetFromMemoryData(*reg_info, &fake_reg[0], + reg_info->byte_size, eByteOrderLittle, + error); + return error; + } + + // For Z registers, zero extend the 128-bit FP register to Z register + // size. + + error = ReadFPR(); + if (error.Fail()) + return error; + + // As we told the client we have Z registers, our own internal offsets + // are set as if we were using an SVE context. We need to work out + // an offset within the FP context instead: + // struct user_fpsimd_state { + // __uint128_t vregs[32]; + // __u32 fpsr; + // __u32 fpcr; + // __u32 __reserved[2]; + // }; + const uint32_t z_num = reg - GetRegisterInfo().GetRegNumSVEZ0(); + offset = z_num * 16; + assert(offset < GetFPRSize()); + src = (uint8_t *)GetFPRBuffer() + offset; + + // Copy from FP into a fake Z value. + std::vector fake_z(reg_info->byte_size, 0); + std::memcpy(&fake_z[0], src, 16 /* 128 bits */); + reg_value.SetFromMemoryData(*reg_info, &fake_z[0], reg_info->byte_size, + eByteOrderLittle, error); + + return error; } else { // SVE enabled, we will read and cache SVE ptrace data error = ReadAllSVE(); @@ -495,13 +544,17 @@ Status NativeRegisterContextLinux_arm64::WriteRegister( return WriteGPR(); } else if (IsFPR(reg)) { - if (m_sve_state == SVEState::Disabled) { - // SVE is disabled take legacy route for FPU register access + if (m_sve_state == SVEState::Disabled || + m_sve_state == SVEState::StreamingFPSIMD) { + // SVE is not present, or we only have it in streaming mode and are + // currently outside of streaming mode. Take normal route for FPU register + // access. error = ReadFPR(); if (error.Fail()) return error; - offset = CalculateFprOffset(reg_info); + offset = CalculateFprOffset(reg_info, + m_sve_state == SVEState::StreamingFPSIMD); assert(offset < GetFPRSize()); dst = (uint8_t *)GetFPRBuffer() + offset; ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size); @@ -544,9 +597,50 @@ Status NativeRegisterContextLinux_arm64::WriteRegister( return WriteAllSVE(); } } else if (IsSVE(reg)) { - if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown) + if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown) { return Status::FromErrorString("SVE disabled or not supported"); - else { + } else if (m_sve_state == SVEState::StreamingFPSIMD) { + // When a target has SVE (in any state), the client is told that it has + // real SVE registers and that the FP registers are just subregisters + // of those SVE registers. This means that any FP write will be converted + // into an SVE write. + // + // If we get here, it did that, but we are outside of streaming mode + // on an SME only system. Meaning there's no way at all to write to actual + // SVE registers. + // + // Instead we will extract the bottom 128 bits of the register, + // write that via the standard FP route and then return the fake SVE + // values as usual. + // + // We can only do this for Z registers. P, FFR and VG have no SIMD + // equivalent. + if (GetRegisterInfo().IsSVERegVG(reg) || + GetRegisterInfo().IsSVEPReg(reg) || + GetRegisterInfo().IsSVERegFFR(reg)) + return Status::FromErrorString( + "Cannot write SVE VG, P or FFR registers while outside of " + "streaming mode."); + + // We have told the client that we only have Z registers and the V + // registers are subsets of Z. This means that the V byte offsets are + // actually for the SVE register context, which we cannot access right + // now. That is, v0 is offset 16, v1 is 16+vlen, and so on. So we will + // manually patch this data into the FP context and write it. + error = ReadFPR(); + if (error.Fail()) + return error; + + uint32_t z_num = reg - GetRegisterInfo().GetRegNumSVEZ0(); + offset = z_num * 16; + assert(offset < GetFPRSize()); + dst = (uint8_t *)GetFPRBuffer() + offset; + // If we get here we must have a Z register. Assume we have 16 bytes aka + // 128 bits at least, enough to fill an FP V register. + ::memcpy(dst, reg_value.GetBytes(), 16); + + return WriteFPR(); + } else { // Target has SVE enabled, we will read and cache SVE ptrace data error = ReadAllSVE(); if (error.Fail()) @@ -696,16 +790,17 @@ Status NativeRegisterContextLinux_arm64::WriteRegister( } enum RegisterSetType : uint32_t { - GPR, - SVE, // Used for SVE and SSVE. - FPR, // When there is no SVE, or SVE in FPSIMD mode. + GPR, // General purpose registers. + SVE, // Used for SVE registers in streaming or non-streaming mode. + FPR, // When there is no SVE, or SVE in FPSIMD mode, or streaming only SVE + // that is in non-streaming mode. // Pointer authentication registers are read only, so not included here. - MTE, - TLS, + MTE, // Memory tagging control registers. + TLS, // Thread local storage registers. SME, // ZA only, because SVCR and SVG are pseudo registers. SME2, // ZT only. - FPMR, - GCS, // Guarded Control Stack registers. + FPMR, // Floating point mode control registers. + GCS, // Guarded Control Stack registers. }; static uint8_t *AddRegisterSetType(uint8_t *dst, @@ -766,8 +861,11 @@ NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) { } } - // If SVE is enabled we need not copy FPR separately. - if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) { + // If SVE is enabled we need not copy FPR separately. If we are in + // non-streaming mode of a streaming only process, then we need to save FPR + // though. + if ((GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) && + m_sve_state != SVEState::StreamingFPSIMD) { // Store mode and register data. cached_size += sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize(); @@ -869,7 +967,8 @@ Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues( m_za_header.size); } - if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) { + if ((GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) && + m_sve_state != SVEState::StreamingFPSIMD) { dst = AddRegisterSetType(dst, RegisterSetType::SVE); *(reinterpret_cast(dst)) = m_sve_state; dst += sizeof(m_sve_state); @@ -1011,11 +1110,58 @@ Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues( GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid, std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this)); break; - case RegisterSetType::FPR: - error = RestoreRegisters( - GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid, - std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this)); + case RegisterSetType::FPR: { + if (!GetRegisterInfo().IsSVEPresent() && + GetRegisterInfo().IsSSVEPresent()) { + // On an SME only system, if we get here then we were outside of + // streaming mode when the registers were saved. We may be in streaming + // mode at the current moment, so we need to to exit it. The kernel + // allows us to do this by writing FPSIMD format data to the + // non-streaming SVE register set, with a vector length of 0 set. This + // is only done in this specific situation. + + size_t data_size = sve::ptrace_fpsimd_offset + GetFPRSize(); + // NT_ARM_SVE data must be a multiple of 128 bits, and the FPU data size + // is not, round up. + data_size = + (data_size + sve::vq_bytes - 1) / sve::vq_bytes * sve::vq_bytes; + std::vector sve_fpsimd_data(data_size); + + user_sve_header *header = + reinterpret_cast(sve_fpsimd_data.data()); + std::memset(header, 0, sizeof(user_sve_header)); + header->size = sve_fpsimd_data.size(); + // VL = 0 tells the process to exit streaming mode. + header->vl = 0; + header->flags = sve::ptrace_regs_fpsimd; + std::memcpy(&sve_fpsimd_data[sve::ptrace_fpsimd_offset], src, + GetFPRSize()); + + struct iovec ioVec; + ioVec.iov_base = sve_fpsimd_data.data(); + ioVec.iov_len = sve_fpsimd_data.size(); + + // Always use non-streaming SVE here, even if the system only + // has streaming mode SVE. + error = WriteRegisterSet(&ioVec, sve_fpsimd_data.size(), NT_ARM_SVE); + + // Wrote FPU, and SVE overlaps FPU. + m_fpu_is_valid = false; + m_sve_buffer_is_valid = false; + m_sve_header_is_valid = false; + + m_sve_state = SVEState::Unknown; + ConfigureRegisterContext(); + + // Consume FP register set. + src += GetFPRSize(); + } else { + error = RestoreRegisters( + GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid, + std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this)); + } break; + } case RegisterSetType::MTE: error = RestoreRegisters( GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid, @@ -1210,7 +1356,6 @@ Status NativeRegisterContextLinux_arm64::ReadFPR() { ioVec.iov_len = GetFPRSize(); error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET); - if (error.Success()) m_fpu_is_valid = true; @@ -1227,6 +1372,9 @@ Status NativeRegisterContextLinux_arm64::WriteFPR() { ioVec.iov_len = GetFPRSize(); m_fpu_is_valid = false; + // SVE Z registers overlap the FP registers. + m_sve_buffer_is_valid = false; + m_sve_header_is_valid = false; return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET); } @@ -1250,7 +1398,13 @@ void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() { } unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() { - return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE; + switch (m_sve_state) { + case SVEState::Streaming: + case SVEState::StreamingFPSIMD: + return NT_ARM_SSVE; + default: + return NT_ARM_SVE; + } } Status NativeRegisterContextLinux_arm64::ReadSVEHeader() { @@ -1598,38 +1752,55 @@ void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() { // If m_sve_state is set to SVEState::Disabled on first stop, code below will // be deemed non operational for the lifetime of current process. if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) { - // If we have SVE we may also have the SVE streaming mode that SME added. - // We can read the header of either mode, but only the active mode will - // have valid register data. + // Systems may have SVE and/or SME. If they are SME only, the SVE regset + // cannot be read from but the SME one can. If they have both SVE and SME, + // only the active mode will return valid register data. - // Check whether SME is present and the streaming SVE mode is active. m_sve_header_is_valid = false; m_sve_buffer_is_valid = false; m_sve_state = SVEState::Streaming; Status error = ReadSVEHeader(); - // Streaming mode is active if the header has the SVE active flag set. - if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) == - sve::ptrace_regs_sve))) { - // Non-streaming might be active instead. + bool has_sme = error.Success(); + bool sme_is_active = + has_sme && + ((m_sve_header.flags & sve::ptrace_regs_mask) == sve::ptrace_regs_sve); + + m_sve_header_is_valid = false; + m_sve_buffer_is_valid = false; + m_sve_state = SVEState::Full; + error = ReadSVEHeader(); + + bool has_sve = error.Success(); + bool sve_is_active = + has_sve && + ((m_sve_header.flags & sve::ptrace_regs_mask) == sve::ptrace_regs_sve); + // We do not check this for streaming mode because the streaming mode regset + // will never be in FP format. + bool fp_is_active = + has_sve && ((m_sve_header.flags & sve::ptrace_regs_mask) == + sve::ptrace_regs_fpsimd); + + if (sme_is_active) + m_sve_state = SVEState::Streaming; + else if (sve_is_active) + m_sve_state = SVEState::Full; + else if (fp_is_active) + m_sve_state = SVEState::FPSIMD; + else if (has_sme) { + // We are in the non-streaming mode of an SME only system. + m_sve_state = SVEState::StreamingFPSIMD; + } else + m_sve_state = SVEState::Disabled; + + if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD || + m_sve_state == SVEState::Streaming || + m_sve_state == SVEState::StreamingFPSIMD) { + m_sve_header_is_valid = false; m_sve_buffer_is_valid = false; - m_sve_state = SVEState::Full; error = ReadSVEHeader(); - if (error.Success()) { - // If SVE is enabled thread can switch between SVEState::FPSIMD and - // SVEState::Full on every stop. - if ((m_sve_header.flags & sve::ptrace_regs_mask) == - sve::ptrace_regs_fpsimd) - m_sve_state = SVEState::FPSIMD; - // Else we are in SVEState::Full. - } else { - m_sve_state = SVEState::Disabled; - } - } - if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD || - m_sve_state == SVEState::Streaming) { // On every stop we configure SVE vector length by calling // ConfigureVectorLengthSVE regardless of current SVEState of this thread. uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE; @@ -1656,8 +1827,30 @@ void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() { } uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset( - const RegisterInfo *reg_info) const { - return reg_info->byte_offset - GetGPRSize(); + const RegisterInfo *reg_info, bool streaming_fpsimd) const { + uint32_t offset = reg_info->byte_offset - GetGPRSize(); + if (!streaming_fpsimd) + return offset; + + // If we're outside of streaming mode on a streaming only target, the offsets + // are relative to an SVE context. We need the offset into the actual FPR + // context: + // struct user_fpsimd_state { + // __uint128_t vregs[32]; + // __u32 fpsr; + // __u32 fpcr; + // __u32 __reserved[2]; + // }; + const size_t fpsr_offset = 16 * 32; + const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB]; + if (reg == GetRegisterInfo().GetRegNumFPSR()) + offset = fpsr_offset; + else if (reg == GetRegisterInfo().GetRegNumFPCR()) + offset = fpsr_offset + 4; + else + offset = 16 * (reg - GetRegisterInfo().GetRegNumFPV0()); + + return offset; } uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset( diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h index 7ed0da8503496..56e29fb31442a 100644 --- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h +++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h @@ -254,7 +254,8 @@ class NativeRegisterContextLinux_arm64 llvm::Error WriteHardwareDebugRegs(DREGType hwbType) override; - uint32_t CalculateFprOffset(const RegisterInfo *reg_info) const; + uint32_t CalculateFprOffset(const RegisterInfo *reg_info, + bool streaming_fpsimd) const; RegisterInfoPOSIX_arm64 &GetRegisterInfo() const; diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp index 3b8d6a84c964c..170a3933bca35 100644 --- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp +++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp @@ -559,6 +559,10 @@ bool RegisterInfoPOSIX_arm64::IsSVERegVG(unsigned reg) const { return sve_vg == reg; } +bool RegisterInfoPOSIX_arm64::IsSVERegFFR(unsigned reg) const { + return sve_ffr == reg; +} + bool RegisterInfoPOSIX_arm64::IsSMERegZA(unsigned reg) const { return reg == m_sme_regnum_collection[2]; } @@ -601,6 +605,8 @@ uint32_t RegisterInfoPOSIX_arm64::GetRegNumFPCR() const { return fpu_fpcr; } uint32_t RegisterInfoPOSIX_arm64::GetRegNumFPSR() const { return fpu_fpsr; } +uint32_t RegisterInfoPOSIX_arm64::GetRegNumFPV0() const { return fpu_v0; } + uint32_t RegisterInfoPOSIX_arm64::GetRegNumSVEVG() const { return sve_vg; } uint32_t RegisterInfoPOSIX_arm64::GetRegNumSMESVG() const { diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h index bca5bff24bff0..4e76ad7f07263 100644 --- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h +++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h @@ -15,7 +15,20 @@ #include "lldb/lldb-private.h" #include -enum class SVEState : uint8_t { Unknown, Disabled, FPSIMD, Full, Streaming }; +enum class SVEState : uint8_t { + // We have yet to look what features there are. + Unknown, + // We know that there is no SVE or streaming SVE (SME). + Disabled, + // We are in non-streaming mode but SVE is not active. + FPSIMD, + // We are in non-streaming mode and SVE is active. + Full, + // We are in streaming mode using streaming SVE. + Streaming, + // We are in non-streaming mode, and only have SVE while in streaming mode. + StreamingFPSIMD +}; class RegisterInfoPOSIX_arm64 : public lldb_private::RegisterInfoAndSetInterface { @@ -143,6 +156,7 @@ class RegisterInfoPOSIX_arm64 bool IsSVEZReg(unsigned reg) const; bool IsSVEPReg(unsigned reg) const; bool IsSVERegVG(unsigned reg) const; + bool IsSVERegFFR(unsigned reg) const; bool IsPAuthReg(unsigned reg) const; bool IsMTEReg(unsigned reg) const; bool IsTLSReg(unsigned reg) const; @@ -156,6 +170,7 @@ class RegisterInfoPOSIX_arm64 uint32_t GetRegNumSVEFFR() const; uint32_t GetRegNumFPCR() const; uint32_t GetRegNumFPSR() const; + uint32_t GetRegNumFPV0() const; uint32_t GetRegNumSVEVG() const; uint32_t GetRegNumSMESVG() const; uint32_t GetPAuthOffset() const;