|
8 | 8 | #include <linux/init.h>
|
9 | 9 | #include <linux/pm.h>
|
10 | 10 | #include <asm/sbi.h>
|
| 11 | +#include <asm/smp.h> |
11 | 12 |
|
12 | 13 | /* default SBI version is 0.1 */
|
13 | 14 | unsigned long sbi_spec_version = SBI_SPEC_VERSION_DEFAULT;
|
@@ -191,6 +192,153 @@ static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask,
|
191 | 192 | }
|
192 | 193 | #endif /* CONFIG_RISCV_SBI_V01 */
|
193 | 194 |
|
| 195 | +static void __sbi_set_timer_v02(uint64_t stime_value) |
| 196 | +{ |
| 197 | +#if __riscv_xlen == 32 |
| 198 | + sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, |
| 199 | + stime_value >> 32, 0, 0, 0, 0); |
| 200 | +#else |
| 201 | + sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, |
| 202 | + 0, 0, 0, 0); |
| 203 | +#endif |
| 204 | +} |
| 205 | + |
| 206 | +static int __sbi_send_ipi_v02(const unsigned long *hart_mask) |
| 207 | +{ |
| 208 | + unsigned long hartid, hmask_val, hbase; |
| 209 | + struct cpumask tmask; |
| 210 | + struct sbiret ret = {0}; |
| 211 | + int result; |
| 212 | + |
| 213 | + if (!hart_mask || !(*hart_mask)) { |
| 214 | + riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); |
| 215 | + hart_mask = cpumask_bits(&tmask); |
| 216 | + } |
| 217 | + |
| 218 | + hmask_val = 0; |
| 219 | + hbase = 0; |
| 220 | + for_each_set_bit(hartid, hart_mask, NR_CPUS) { |
| 221 | + if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { |
| 222 | + ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, |
| 223 | + hmask_val, hbase, 0, 0, 0, 0); |
| 224 | + if (ret.error) |
| 225 | + goto ecall_failed; |
| 226 | + hmask_val = 0; |
| 227 | + hbase = 0; |
| 228 | + } |
| 229 | + if (!hmask_val) |
| 230 | + hbase = hartid; |
| 231 | + hmask_val |= 1UL << (hartid - hbase); |
| 232 | + } |
| 233 | + |
| 234 | + if (hmask_val) { |
| 235 | + ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, |
| 236 | + hmask_val, hbase, 0, 0, 0, 0); |
| 237 | + if (ret.error) |
| 238 | + goto ecall_failed; |
| 239 | + } |
| 240 | + |
| 241 | + return 0; |
| 242 | + |
| 243 | +ecall_failed: |
| 244 | + result = sbi_err_map_linux_errno(ret.error); |
| 245 | + pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", |
| 246 | + __func__, hbase, hmask_val, result); |
| 247 | + return result; |
| 248 | +} |
| 249 | + |
| 250 | +static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, |
| 251 | + unsigned long hbase, unsigned long start, |
| 252 | + unsigned long size, unsigned long arg4, |
| 253 | + unsigned long arg5) |
| 254 | +{ |
| 255 | + struct sbiret ret = {0}; |
| 256 | + int ext = SBI_EXT_RFENCE; |
| 257 | + int result = 0; |
| 258 | + |
| 259 | + switch (fid) { |
| 260 | + case SBI_EXT_RFENCE_REMOTE_FENCE_I: |
| 261 | + ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0); |
| 262 | + break; |
| 263 | + case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: |
| 264 | + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 265 | + size, 0, 0); |
| 266 | + break; |
| 267 | + case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: |
| 268 | + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 269 | + size, arg4, 0); |
| 270 | + break; |
| 271 | + |
| 272 | + case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: |
| 273 | + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 274 | + size, 0, 0); |
| 275 | + break; |
| 276 | + case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: |
| 277 | + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 278 | + size, arg4, 0); |
| 279 | + break; |
| 280 | + case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: |
| 281 | + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 282 | + size, 0, 0); |
| 283 | + break; |
| 284 | + case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: |
| 285 | + ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 286 | + size, arg4, 0); |
| 287 | + break; |
| 288 | + default: |
| 289 | + pr_err("unknown function ID [%lu] for SBI extension [%d]\n", |
| 290 | + fid, ext); |
| 291 | + result = -EINVAL; |
| 292 | + } |
| 293 | + |
| 294 | + if (ret.error) { |
| 295 | + result = sbi_err_map_linux_errno(ret.error); |
| 296 | + pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", |
| 297 | + __func__, hbase, hmask_val, result); |
| 298 | + } |
| 299 | + |
| 300 | + return result; |
| 301 | +} |
| 302 | + |
| 303 | +static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, |
| 304 | + unsigned long start, unsigned long size, |
| 305 | + unsigned long arg4, unsigned long arg5) |
| 306 | +{ |
| 307 | + unsigned long hmask_val, hartid, hbase; |
| 308 | + struct cpumask tmask; |
| 309 | + int result; |
| 310 | + |
| 311 | + if (!hart_mask || !(*hart_mask)) { |
| 312 | + riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); |
| 313 | + hart_mask = cpumask_bits(&tmask); |
| 314 | + } |
| 315 | + |
| 316 | + hmask_val = 0; |
| 317 | + hbase = 0; |
| 318 | + for_each_set_bit(hartid, hart_mask, NR_CPUS) { |
| 319 | + if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { |
| 320 | + result = __sbi_rfence_v02_call(fid, hmask_val, hbase, |
| 321 | + start, size, arg4, arg5); |
| 322 | + if (result) |
| 323 | + return result; |
| 324 | + hmask_val = 0; |
| 325 | + hbase = 0; |
| 326 | + } |
| 327 | + if (!hmask_val) |
| 328 | + hbase = hartid; |
| 329 | + hmask_val |= 1UL << (hartid - hbase); |
| 330 | + } |
| 331 | + |
| 332 | + if (hmask_val) { |
| 333 | + result = __sbi_rfence_v02_call(fid, hmask_val, hbase, |
| 334 | + start, size, arg4, arg5); |
| 335 | + if (result) |
| 336 | + return result; |
| 337 | + } |
| 338 | + |
| 339 | + return 0; |
| 340 | +} |
| 341 | + |
194 | 342 | /**
|
195 | 343 | * sbi_set_timer() - Program the timer for next timer event.
|
196 | 344 | * @stime_value: The value after which next timer event should fire.
|
@@ -266,6 +414,85 @@ void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
|
266 | 414 | }
|
267 | 415 | EXPORT_SYMBOL(sbi_remote_sfence_vma_asid);
|
268 | 416 |
|
| 417 | +/** |
| 418 | + * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote |
| 419 | + * harts for the specified guest physical address range. |
| 420 | + * @hart_mask: A cpu mask containing all the target harts. |
| 421 | + * @start: Start of the guest physical address |
| 422 | + * @size: Total size of the guest physical address range. |
| 423 | + * |
| 424 | + * Return: None |
| 425 | + */ |
| 426 | +int sbi_remote_hfence_gvma(const unsigned long *hart_mask, |
| 427 | + unsigned long start, |
| 428 | + unsigned long size) |
| 429 | +{ |
| 430 | + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, |
| 431 | + hart_mask, start, size, 0, 0); |
| 432 | +} |
| 433 | +EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); |
| 434 | + |
| 435 | +/** |
| 436 | + * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given |
| 437 | + * remote harts for a guest physical address range belonging to a specific VMID. |
| 438 | + * |
| 439 | + * @hart_mask: A cpu mask containing all the target harts. |
| 440 | + * @start: Start of the guest physical address |
| 441 | + * @size: Total size of the guest physical address range. |
| 442 | + * @vmid: The value of guest ID (VMID). |
| 443 | + * |
| 444 | + * Return: 0 if success, Error otherwise. |
| 445 | + */ |
| 446 | +int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, |
| 447 | + unsigned long start, |
| 448 | + unsigned long size, |
| 449 | + unsigned long vmid) |
| 450 | +{ |
| 451 | + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, |
| 452 | + hart_mask, start, size, vmid, 0); |
| 453 | +} |
| 454 | +EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); |
| 455 | + |
| 456 | +/** |
| 457 | + * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote |
| 458 | + * harts for the current guest virtual address range. |
| 459 | + * @hart_mask: A cpu mask containing all the target harts. |
| 460 | + * @start: Start of the current guest virtual address |
| 461 | + * @size: Total size of the current guest virtual address range. |
| 462 | + * |
| 463 | + * Return: None |
| 464 | + */ |
| 465 | +int sbi_remote_hfence_vvma(const unsigned long *hart_mask, |
| 466 | + unsigned long start, |
| 467 | + unsigned long size) |
| 468 | +{ |
| 469 | + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, |
| 470 | + hart_mask, start, size, 0, 0); |
| 471 | +} |
| 472 | +EXPORT_SYMBOL(sbi_remote_hfence_vvma); |
| 473 | + |
| 474 | +/** |
| 475 | + * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given |
| 476 | + * remote harts for current guest virtual address range belonging to a specific |
| 477 | + * ASID. |
| 478 | + * |
| 479 | + * @hart_mask: A cpu mask containing all the target harts. |
| 480 | + * @start: Start of the current guest virtual address |
| 481 | + * @size: Total size of the current guest virtual address range. |
| 482 | + * @asid: The value of address space identifier (ASID). |
| 483 | + * |
| 484 | + * Return: None |
| 485 | + */ |
| 486 | +int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, |
| 487 | + unsigned long start, |
| 488 | + unsigned long size, |
| 489 | + unsigned long asid) |
| 490 | +{ |
| 491 | + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, |
| 492 | + hart_mask, start, size, asid, 0); |
| 493 | +} |
| 494 | +EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); |
| 495 | + |
269 | 496 | /**
|
270 | 497 | * sbi_probe_extension() - Check if an SBI extension ID is supported or not.
|
271 | 498 | * @extid: The extension ID to be probed.
|
@@ -332,11 +559,29 @@ int __init sbi_init(void)
|
332 | 559 | if (!sbi_spec_is_0_1()) {
|
333 | 560 | pr_info("SBI implementation ID=0x%lx Version=0x%lx\n",
|
334 | 561 | sbi_get_firmware_id(), sbi_get_firmware_version());
|
| 562 | + if (sbi_probe_extension(SBI_EXT_TIME) > 0) { |
| 563 | + __sbi_set_timer = __sbi_set_timer_v02; |
| 564 | + pr_info("SBI v0.2 TIME extension detected\n"); |
| 565 | + } else { |
| 566 | + __sbi_set_timer = __sbi_set_timer_v01; |
| 567 | + } |
| 568 | + if (sbi_probe_extension(SBI_EXT_IPI) > 0) { |
| 569 | + __sbi_send_ipi = __sbi_send_ipi_v02; |
| 570 | + pr_info("SBI v0.2 IPI extension detected\n"); |
| 571 | + } else { |
| 572 | + __sbi_send_ipi = __sbi_send_ipi_v01; |
| 573 | + } |
| 574 | + if (sbi_probe_extension(SBI_EXT_RFENCE) > 0) { |
| 575 | + __sbi_rfence = __sbi_rfence_v02; |
| 576 | + pr_info("SBI v0.2 RFENCE extension detected\n"); |
| 577 | + } else { |
| 578 | + __sbi_rfence = __sbi_rfence_v01; |
| 579 | + } |
| 580 | + } else { |
| 581 | + __sbi_set_timer = __sbi_set_timer_v01; |
| 582 | + __sbi_send_ipi = __sbi_send_ipi_v01; |
| 583 | + __sbi_rfence = __sbi_rfence_v01; |
335 | 584 | }
|
336 | 585 |
|
337 |
| - __sbi_set_timer = __sbi_set_timer_v01; |
338 |
| - __sbi_send_ipi = __sbi_send_ipi_v01; |
339 |
| - __sbi_rfence = __sbi_rfence_v01; |
340 |
| - |
341 | 586 | return 0;
|
342 | 587 | }
|
0 commit comments