SuperTinyKernel™ RTOS 1.08.x
Lightweight, high-performance, deterministic, bare-metal C++ RTOS for resource-constrained embedded systems. MIT Open Source License.
Loading...
Searching...
No Matches
stk_arch_risc-v.cpp
Go to the documentation of this file.
1/*
2 * SuperTinyKernel(TM) RTOS: Lightweight High-Performance Deterministic C++ RTOS for Embedded Systems.
3 *
4 * Source: https://github.com/SuperTinyKernel-RTOS
5 *
6 * Copyright (c) 2022-2026 Neutron Code Limited <stk@neutroncode.com>. All Rights Reserved.
7 * License: MIT License, see LICENSE for a full text.
8 */
9
10// note: If missing, this header must be customized (get it in the root of the source folder) and
11// copied to the /include folder manually.
12#include "stk_config.h"
13
14#ifdef _STK_ARCH_RISC_V
15
16#include "stk_arch.h"
18#include "stk_helper.h"
19
20using namespace stk;
21
60//#define _STK_RISCV_USE_PENDSV
61
62// CLINT
63// Details: https://github.com/riscv/riscv-aclint/blob/main/riscv-aclint.adoc
64#ifndef STK_RISCV_CLINT_BASE_ADDR
65 #define STK_RISCV_CLINT_BASE_ADDR (0x2000000U)
66#endif
67#ifndef STK_RISCV_CLINT_MTIMECMP_ADDR
68 #define STK_RISCV_CLINT_MTIMECMP_ADDR (STK_RISCV_CLINT_BASE_ADDR + 0x4000U) // 8-byte value, 1 per hart
69#endif
70#ifndef STK_RISCV_CLINT_MTIME_ADDR
71 #define STK_RISCV_CLINT_MTIME_ADDR (STK_RISCV_CLINT_BASE_ADDR + 0xBFF8U) // 8-byte value, global
72#endif
73
78#define STK_RISCV_ISR_STACK_SIZE 256U
79
83#ifndef STK_TIMER_CLOCK_FREQUENCY
84 #define STK_TIMER_CLOCK_FREQUENCY 1000000U
85#endif
86
87// Spin lock timeout.
88#ifndef STK_SPINLOCK_TIMEOUT_US
89 #define STK_SPINLOCK_TIMEOUT_US (5U * 1000U * 1000U) // 5 sec
90#endif
91
92// Conservative lower bound on cycles consumed per iteration of the HW_SpinLockLock retry
93// loop (atomic test-and-set attempt + compare + branch + relax hint). Deliberately an
94// underestimate, so the derived iteration budget always waits at least
95// STK_SPINLOCK_TIMEOUT_US before giving up rather than timing out early. If
96// __stk_relax_cpu() ever becomes a multi-cycle instruction (or the pause/fence it lowers
97// to on a given target is costlier) on some core, raise this.
98#ifndef STK_SPINLOCK_MIN_CYCLES_PER_ITER
99 #define STK_SPINLOCK_MIN_CYCLES_PER_ITER (4U)
100#endif
101
106#ifndef STK_RISCV_ISR_SECTION
107 #define STK_RISCV_ISR_SECTION
108#endif
109
114#define STK_RISCV_ISR extern "C" STK_RISCV_ISR_SECTION __attribute__ ((interrupt("machine")))
115
117#define STK_ASM_EXIT_FROM_HANDLER "mret"
118
124#ifndef STK_RISCV_CLINT_MTIMECMP_PER_HART
125 #define STK_RISCV_CLINT_MTIMECMP_PER_HART 1
126#endif
127
132#ifndef STK_ARCH_GET_CPU_ID
133 #define STK_ARCH_GET_CPU_ID() read_csr(mhartid)
134#endif
135
137#if (__riscv_flen == 0)
138 #define STK_RISCV_FPU 0
139#else
140 #define STK_RISCV_FPU __riscv_flen
141#endif
142
143#define STR(x) #x
144#define XSTR(s) STR(s)
145
147#if (__riscv_xlen == 32)
148 #define REGBYTES XSTR(4)
149 #define LREG XSTR(lw)
150 #define SREG XSTR(sw)
151#elif (__riscv_xlen == 64)
152 #define REGBYTES XSTR(8)
153 #define LREG XSTR(ld)
154 #define SREG XSTR(sd)
155#else
156 #error Unsupported RISC-V platform!
157#endif
158
159#if (STK_RISCV_FPU == 32)
160 #define FREGBYTES XSTR(4)
161 #define FLREG XSTR(flw)
162 #define FSREG XSTR(fsw)
163#elif (STK_RISCV_FPU == 64)
164 #define FREGBYTES XSTR(8)
165 #define FLREG XSTR(fld)
166 #define FSREG XSTR(fsd)
167#elif (STK_RISCV_FPU != 0)
168#error Unsupported FP register count!
169#endif
170
171
172#if (__riscv_32e == 1)
173 #define STK_RISCV_REGISTER_COUNT (15U + (STK_RISCV_FPU != 0U ? 31U : 0U))
174#else
175 #define STK_RISCV_REGISTER_COUNT (31U + (STK_RISCV_FPU != 0U ? 31U : 0U))
176#endif
177
178#define STK_SERVICE_SLOTS 2U // (0) mepc, (1) mstatus
179
180#if (__riscv_32e == 1)
181 #define FOFFSET XSTR(68) // FP stack offset = (17 * 4)
182 #if (STK_RISCV_FPU == 0)
183 #define REGSIZE XSTR(((15U + STK_SERVICE_SLOTS) * 4U)) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus
184 #else
185 #if (STK_RISCV_FPU == 32)
186 #define REGSIZE XSTR((((15U + STK_SERVICE_SLOTS) * 4U) + (31U * 4U))) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus + 32 fp registers
187 #elif (STK_RISCV_FPU == 64)
188 #define REGSIZE XSTR((((15U + STK_SERVICE_SLOTS) * 4U) + (31U * 8U))) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus + 32 fp registers
189 #endif
190 #endif
191#elif (__riscv_xlen == 32)
192 #define FOFFSET XSTR(132) // FP stack offset = (33 * 4)
193 #if (STK_RISCV_FPU == 0)
194 #define REGSIZE XSTR(((31U + STK_SERVICE_SLOTS) * 4U)) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus
195 #else
196 #if (STK_RISCV_FPU == 32)
197 #define REGSIZE XSTR((((31U + STK_SERVICE_SLOTS) * 4U) + (31U * 4U))) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus + 32 fp registers
198 #elif (STK_RISCV_FPU == 64)
199 #define REGSIZE XSTR((((31U + STK_SERVICE_SLOTS) * 4U) + (31U * 8U))) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus + 32 fp registers
200 #endif
201 #endif
202#elif (__riscv_xlen == 64)
203 #define FOFFSET XSTR(264) // FP stack offset = (33 * 8)
204 #if (STK_RISCV_FPU == 0)
205 #define REGSIZE XSTR(((31U + STK_SERVICE_SLOTS) * 8U)) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus
206 #else
207 #if (STK_RISCV_FPU == 32)
208 #define REGSIZE XSTR((((31U + STK_SERVICE_SLOTS) * 8U) + (31U * 4U))) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus + 32 fp registers
209 #elif (STK_RISCV_FPU == 64)
210 #define REGSIZE XSTR((((31U + STK_SERVICE_SLOTS) * 8U) + (31U * 8U))) // STK_RISCV_REGISTER_COUNT + 2 for mepc, mstatus + 32 fp registers
211 #endif
212 #endif
213#endif
214
215#if (__riscv_xlen == 32)
216 #define REGBYTES_LOG2 "2" // log2(4) - used for hart-index shift
217#elif (__riscv_xlen == 64)
218 #define REGBYTES_LOG2 "3" // log2(8)
219#endif
220
227#ifndef STK_SYSTICK_HANDLER
228 #define STK_SYSTICK_HANDLER riscv_mtvec_mti
229#endif
230
237#ifndef STK_SVC_HANDLER
238 #define STK_SVC_HANDLER riscv_mtvec_exception
239#endif
240
248#ifndef STK_MSI_HANDLER
249 #define STK_MSI_HANDLER riscv_mtvec_msi
250#endif
251
252// ----------------------------------------------------------------------------
253namespace {
254// ----------------------------------------------------------------------------
255
274struct TaskFrame
275{
276 // Service slots (indices 0, 1) - sit at sp+0 and sp+REGBYTES
277 Word MEPC;
278 Word MSTATUS;
279
280 // General-purpose register slots (indices 2..N), one per xN
281 Word X1_RA;
282 Word X2_SP;
283#if (STK_RISCV_FPU != 0)
284 Word X3_FSR;
285#else
286 Word X3_GP;
287#endif
288 Word X4;
289 Word X5;
290 Word X6;
291 Word X7;
292 Word X8;
293 Word X9;
294 Word X10_A0;
295 Word X11;
296 Word X12;
297 Word X13;
298 Word X14;
299 Word X15;
300#if (__riscv_32e != 1)
301 Word X16;
302 Word X17;
303 Word X18;
304 Word X19;
305 Word X20;
306 Word X21;
307 Word X22;
308 Word X23;
309 Word X24;
310 Word X25;
311 Word X26;
312 Word X27;
313 Word X28;
314 Word X29;
315 Word X30;
316 Word X31;
317#endif
318#if (STK_RISCV_FPU != 0)
319 // FP register slots - at FOFFSET from frame base, immediately after integer slots.
320 // FREGBYTES may differ from REGBYTES (32-bit FP on a 64-bit integer machine).
321 // Declared as Word arrays for uniform struct sizing; the FP load/store
322 // instructions address them by byte offset and tolerate the type mismatch.
323 Word F[32];
324#endif
325};
326
327// ----------------------------------------------------------------------------
328} // namespace
329// ----------------------------------------------------------------------------
330
331// Local functions of the driver:
332// ----------------------------------------------------------------------------
333namespace {
334namespace drv {
335// ----------------------------------------------------------------------------
336
337void OnTaskRun(ITask *task);
338void OnTaskExit();
339void OnSchedulerSleep();
340void OnSchedulerSleepOverride();
341void OnSchedulerExit();
342
343// ----------------------------------------------------------------------------
344} // namespace drv
345} // namespace
346// ----------------------------------------------------------------------------
347
359extern "C" {
360
364extern Stack *volatile s_StkRiscvStackIdle[STK_ARCH_CPU_COUNT];
365
369extern Stack *volatile s_StkRiscvStackActive[STK_ARCH_CPU_COUNT];
370
374extern Stack *volatile s_StkRiscvStackIsr[STK_ARCH_CPU_COUNT];
375
376#ifdef _STK_RISCV_USE_PENDSV
382extern volatile Word s_StkRiscvSpIsrInt[STK_ARCH_CPU_COUNT];
383#endif
384
389extern volatile EKernelPanicId g_StkLastPanicId;
390
391} // extern "C"
392
395Stack *volatile s_StkRiscvStackIdle[STK_ARCH_CPU_COUNT];
396
399Stack *volatile s_StkRiscvStackActive[STK_ARCH_CPU_COUNT];
400
403Stack *volatile s_StkRiscvStackIsr[STK_ARCH_CPU_COUNT];
404
405#ifdef _STK_RISCV_USE_PENDSV
408volatile Word s_StkRiscvSpIsrInt[STK_ARCH_CPU_COUNT];
409#endif
410
413volatile EKernelPanicId g_StkLastPanicId = KERNEL_PANIC_NONE;
414
415// ----------------------------------------------------------------------------
416namespace {
417// ----------------------------------------------------------------------------
418
419// Local static variables:
421#if STK_CORE_FREQ_UNIFIED
422uint32_t s_StkSystemCoreClock[1U];
423#else
424uint32_t s_StkSystemCoreClock[STK_ARCH_CPU_COUNT];
425#endif
426
436uint32_t s_StkSpinlockTimeoutIters = 0xFFFFFFFU;
437
441volatile bool s_StkRiscvCsuLock = false;
442
445__stk_forceinline void __DSB()
446{
447 __asm volatile("fence rw, rw" ::: "memory");
448}
449
452__stk_forceinline void __ISB()
453{
454#ifdef __riscv_zifencei
455 __asm volatile("fence.i" ::: "memory");
456#else
457 __sync_synchronize();
458#endif
459}
460
463__stk_forceinline void __WFI()
464{
465 __asm volatile("wfi");
466}
467
471__stk_forceinline void HW_StartScheduler()
472{
473 __asm volatile("ecall");
474}
475
478__stk_forceinline uint8_t HW_GetHartId()
479{
480 return STK_ARCH_GET_CPU_ID();
481}
482
485__stk_forceinline void HW_DisableInterrupts()
486{
487 __asm volatile("csrrci zero, mstatus, %0"
488 : /* output: none */
489 : "i"(MSTATUS_MIE)
490 : /* clobbers: none */);
491}
492
495__stk_forceinline void HW_EnableInterrupts()
496{
497 __asm volatile("csrrsi zero, mstatus, %0"
498 : /* output: none */
499 : "i"(MSTATUS_MIE)
500 : /* clobbers: none */);
501}
502
506__stk_forceinline Word HW_EnterCriticalSection()
507{
508 Word ses;
509 __asm volatile("csrrci %0, mstatus, %1"
510 : "=r"(ses)
511 : "i"(MSTATUS_MIE)
512 : /* clobbers: none */);
513
514 return ses;
515}
516
520__stk_forceinline void HW_ExitCriticalSection(Word ses)
521{
522 __asm volatile("csrrs zero, mstatus, %0"
523 : /* output: none */
524 : "r"(ses)
525 : /* clobbers: none */);
526}
527
530__stk_forceinline void HW_EnterSleepMode()
531{
532 __DSB(); // data barrier
533 __WFI(); // enter standby mode until time slot expires
534}
535
538__stk_forceinline void HW_StopMTimer()
539{
540 clear_csr(mie, MIP_MTIP);
541}
542
546__stk_forceinline void HW_ScheduleContextSwitch(uint8_t hart)
547{
548 // Pend Machine Software Interrupt (MSI) - equivalent of ARM's PENDSVSET
549 volatile uint32_t *msip = (volatile uint32_t *)(STK_RISCV_CLINT_BASE_ADDR);
550 msip[hart] = 1U; // set pending
551 __DSB();
552}
553
559__stk_forceinline void HW_ClearPendingSwitch()
560{
561 clear_csr(mie, MIP_MSIP);
562}
563
567__stk_forceinline uint32_t HW_CoreClockFrequency()
568{
569#if STK_CORE_FREQ_UNIFIED
570 return s_StkSystemCoreClock[0];
571#else
572 return s_StkSystemCoreClock[STK_ARCH_GET_CPU_ID()];
573#endif
574}
575
580__stk_forceinline uint32_t HW_MtimeClockFrequency()
581{
582 return STK_TIMER_CLOCK_FREQUENCY; // timer frequency, e.g. 1 MHz
583}
584
590__stk_forceinline uint64_t HW_GetMtime()
591{
592#if ( __riscv_xlen > 32)
593 return *(hw::WordToPtr<volatile uint64_t>(STK_RISCV_CLINT_MTIME_ADDR));
594#else
595 const Word mtime_base = STK_RISCV_CLINT_MTIME_ADDR;
596 const volatile uint32_t *const mtime_lo = hw::WordToPtr<volatile uint32_t>(mtime_base);
597 const volatile uint32_t *const mtime_hi = hw::WordToPtr<volatile uint32_t>(mtime_base + sizeof(uint32_t));
598
599 uint32_t hi, lo;
600 do
601 {
602 hi = (*mtime_hi);
603 lo = (*mtime_lo);
604 }
605 while (hi != (*mtime_hi)); // make sure mtime_hi did not tick when read mtime_lo
606
607 return (static_cast<uint64_t>(hi) << 32) | lo;
608#endif
609}
610
614__stk_forceinline void HW_SetMtimecmp(uint64_t time_next)
615{
616#if STK_RISCV_CLINT_MTIMECMP_PER_HART
617 const uint8_t hart = HW_GetHartId();
618#else
619 const uint8_t hart = 0U;
620#endif
621
622#if (__riscv_xlen == 64)
623 hw::WordToPtr<volatile uint64_t>(STK_RISCV_CLINT_MTIMECMP_ADDR)[hart] = time_next;
624#else
625 const Word mtimecmp_base = STK_RISCV_CLINT_MTIMECMP_ADDR + (hart * sizeof(uint64_t));
626 volatile uint32_t *mtimecmp_lo = hw::WordToPtr<volatile uint32_t>(mtimecmp_base);
627 volatile uint32_t *mtimecmp_hi = hw::WordToPtr<volatile uint32_t>(mtimecmp_base + sizeof(uint32_t));
628
629 // expecting 4-byte aligned memory
630 STK_ASSERT(((uintptr_t)mtimecmp_lo & (4U - 1U)) == 0U);
631 STK_ASSERT(((uintptr_t)mtimecmp_hi & (4U - 1U)) == 0U);
632
633 // prevent unexpected interrupt by setting some very large value to the high part
634 // details: https://riscv.org/wp-content/uploads/2017/05/riscv-privileged-v1.10.pdf, page 31
635 (*mtimecmp_hi) = ~0U;
636
637 (*mtimecmp_lo) = (uint32_t)(time_next & 0xFFFFFFFFU);
638 (*mtimecmp_hi) = (uint32_t)(time_next >> 32);
639#endif
640}
641
647__stk_forceinline uint64_t HW_GetMtimeElapsed(uint64_t since)
648{
649 return HW_GetMtime() - since;
650}
651
654__stk_forceinline void HW_EnableCycleCounter()
655{
656 __asm volatile("csrci mcountinhibit, 0x1");
657}
658
662__stk_forceinline Cycles HW_GetCycleCounter()
663{
664 uint32_t high, low, check;
665 do
666 {
667 __asm volatile("csrr %0, mcycleh" : "=r"(high));
668 __asm volatile("csrr %0, mcycle" : "=r"(low));
669 __asm volatile("csrr %0, mcycleh" : "=r"(check));
670 }
671 while (high != check);
672
673 return (static_cast<Cycles>(high) << 32) | low;
674}
675
678__stk_forceinline Word HW_GetCallerSP()
679{
680 Word sp;
681 __asm volatile("mv %0, sp"
682 : "=r"(sp)
683 : /* input: none */
684 : /* clobbers: none */);
685
686 return sp;
687}
688
691__stk_forceinline void HW_CriticalSectionStart(Word &ses)
692{
693 ses = HW_EnterCriticalSection();
694
695 // ensure the disable is recognized before subsequent code
696 __DSB();
697 __ISB();
698}
699
702__stk_forceinline void HW_CriticalSectionEnd(Word ses)
703{
704 // ensure all memory work is finished before re-enabling
705 __DSB();
706
707 HW_ExitCriticalSection(ses);
708
709 // synchronization point: any pending interrupt can be serviced immediately at this boundary
710 __ISB();
711}
712
727__stk_forceinline bool HW_SpinLockTryLock(volatile bool &lock)
728{
729 return !__atomic_test_and_set(&lock, __ATOMIC_ACQUIRE);
730}
731
752__stk_forceinline void HW_SpinLockLock(volatile bool &lock)
753{
754 uint32_t timeout = s_StkSpinlockTimeoutIters;
755 while (!HW_SpinLockTryLock(lock))
756 {
757 if (--timeout == 0U)
758 {
759 // Invariant violated: the lock owner exited without releasing,
760 // Kernel state is suspect, enter defined safe state.
762 }
763 __stk_relax_cpu();
764 }
765}
766
782__stk_forceinline void HW_SpinLockUnlock(volatile bool &lock)
783{
784 if (!lock)
785 {
786 STK_KERNEL_PANIC(KERNEL_PANIC_SPINLOCK_DEADLOCK); // release attempt of unowned lock
787 }
788
789 // ensure all data writes (like scheduling metadata) are flushed before the lock is released:
790 // __atomic_clear with __ATOMIC_RELEASE provides the required store-release barrier,
791 // the explicit fence rw,w is retained for toolchains that do not lower __ATOMIC_RELEASE
792 // to a full release fence on all RISC-V targets
793 __asm volatile("fence rw, w" ::: "memory");
794
795 __atomic_clear(&lock, __ATOMIC_RELEASE);
796}
797
810void HW_InitSpinlockTimeout()
811{
812 const Cycles cycles_budget = ConvertTimeUsToClockCycles(HW_CoreClockFrequency(), STK_SPINLOCK_TIMEOUT_US);
813 const Cycles iters = cycles_budget / STK_SPINLOCK_MIN_CYCLES_PER_ITER;
814
815 // floor guards against a zero/not-yet-configured clock; ceiling guards against overflow
816 // when narrowing to uint32_t
817 s_StkSpinlockTimeoutIters = static_cast<uint32_t>(
818 stk::Min(stk::Max(iters, static_cast<Cycles>(0x1000U)), static_cast<Cycles>(0xFFFFFFFFU)));
819}
820
821// ----------------------------------------------------------------------------
822namespace drv {
823// ----------------------------------------------------------------------------
824
844
854struct JmpFrame
855{
856 Word RA;
857 Word SP;
858 Word S0;
859 Word S1;
860 Word S2;
861 Word S3;
862 Word S4;
863 Word S5;
864 Word S6;
865 Word S7;
866 Word S8;
867 Word S9;
868 Word S10;
869 Word S11;
870#if (STK_RISCV_FPU != 0)
871 Word FCSR;
872#endif
873};
874
889__attribute__((naked))
890int32_t SaveJmp(JmpFrame &/*f*/)
891{
892 __asm volatile(
893 // a0 = &f - no prologue has touched sp or s0 yet
894 SREG " ra, 0*" REGBYTES "(a0) \n" // save return address
895 SREG " sp, 1*" REGBYTES "(a0) \n" // save caller's stack pointer
896 SREG " s0, 2*" REGBYTES "(a0) \n"
897 SREG " s1, 3*" REGBYTES "(a0) \n"
898 SREG " s2, 4*" REGBYTES "(a0) \n"
899 SREG " s3, 5*" REGBYTES "(a0) \n"
900 SREG " s4, 6*" REGBYTES "(a0) \n"
901 SREG " s5, 7*" REGBYTES "(a0) \n"
902 SREG " s6, 8*" REGBYTES "(a0) \n"
903 SREG " s7, 9*" REGBYTES "(a0) \n"
904 SREG " s8, 10*" REGBYTES "(a0) \n"
905 SREG " s9, 11*" REGBYTES "(a0) \n"
906 SREG " s10, 12*" REGBYTES "(a0) \n"
907 SREG " s11, 13*" REGBYTES "(a0) \n"
908#if (STK_RISCV_FPU != 0)
909 "frcsr t0 \n" // read fcsr (rounding mode + flags)
910 SREG " t0, 14*" REGBYTES "(a0) \n" // save to JmpFrame::FCSR
911#endif
912 "li a0, 0 \n" // return 0
913 "ret \n" // explicit return (naked)
914 );
915}
916
934__attribute__((naked, noreturn))
935void RestoreJmp(JmpFrame &/*f*/, int32_t /*val*/)
936{
937 __asm volatile(
938 // a0 = &f, a1 = val
939 LREG " ra, 0*" REGBYTES "(a0) \n"
940 LREG " sp, 1*" REGBYTES "(a0) \n"
941 LREG " s0, 2*" REGBYTES "(a0) \n"
942 LREG " s1, 3*" REGBYTES "(a0) \n"
943 LREG " s2, 4*" REGBYTES "(a0) \n"
944 LREG " s3, 5*" REGBYTES "(a0) \n"
945 LREG " s4, 6*" REGBYTES "(a0) \n"
946 LREG " s5, 7*" REGBYTES "(a0) \n"
947 LREG " s6, 8*" REGBYTES "(a0) \n"
948 LREG " s7, 9*" REGBYTES "(a0) \n"
949 LREG " s8, 10*" REGBYTES "(a0) \n"
950 LREG " s9, 11*" REGBYTES "(a0) \n"
951 LREG " s10, 12*" REGBYTES "(a0) \n"
952 LREG " s11, 13*" REGBYTES "(a0) \n"
953#if (STK_RISCV_FPU != 0)
954 LREG " t0, 14*" REGBYTES "(a0) \n" // load saved fcsr into t0
955 "fscsr t0 \n" // restore rounding mode + flags
956#endif
957 "mv a0, a1 \n" // return val to SaveJmp's caller
958 "ret \n" // jump to saved RA
959 );
960}
961
962// ----------------------------------------------------------------------------
963} // namespace drv
964// ----------------------------------------------------------------------------
965
966// ----------------------------------------------------------------------------
967} // namespace
968// ----------------------------------------------------------------------------
969
970// ----------------------------------------------------------------------------
971namespace {
972// ----------------------------------------------------------------------------
973
975struct Context final : public PlatformContext
976{
977 constexpr Context() : PlatformContext(), m_stack_main(), m_stack_isr(), m_stack_isr_mem(),
978 m_exit_buf(), m_overrider(nullptr), m_specific(nullptr), m_tick_period(0), m_last_mtime(0ULL),
979 #if STK_TICKLESS_IDLE
980 m_sleep_ticks(0),
981 #endif
982 m_csu(0), m_csu_nesting(0),
983 m_starting(false), m_started(false), m_exiting(false)
984
985 {}
986
987 void Initialize(IPlatform::IEventHandler *handler, IKernelService *service, Stack *exit_trap,
988 uint32_t resolution_us)
989 {
990 InitializeBase(handler, service, exit_trap, resolution_us);
991
992 STK_STATIC_ASSERT_DESC_N(SP, offsetof(Stack, SP) == 0U,
993 "expect Stack::mode member at offset of 0 (first member)");
994 STK_ASSERT(s_StkRiscvCsuLock == false);
995 STK_ASSERT(g_StkLastPanicId == KERNEL_PANIC_NONE);
996
997 // init ISR's stack
998 {
999 StackMemoryWrapper<STK_RISCV_ISR_STACK_SIZE> stack_isr_mem(&m_stack_isr_mem);
1000 m_stack_isr.SP = InitStackMemory(&stack_isr_mem);
1001 m_stack_isr.access_mode = ACCESS_PRIVILEGED;
1002 }
1003
1004 // init Main stack
1005 {
1006 m_stack_main.SP = STK_STACK_MEMORY_FILLER;
1007 m_stack_main.access_mode = ACCESS_PRIVILEGED;
1008 }
1009
1010 m_csu = 0U;
1011 m_csu_nesting = 0U;
1012 m_tick_period = ConvertTimeUsToClockCycles(STK_TIMER_CLOCK_FREQUENCY, resolution_us);
1013 m_last_mtime = 0ULL;
1014 m_starting = false;
1015 m_started = false;
1016 m_exiting = false;
1017
1018 // initialize system clock if user-side code has not yet done it
1019 if (s_StkSystemCoreClock[0] == 0U)
1020 {
1021 for (uint8_t i = 0U; i < STK_STATIC_ARRAY_SIZE(s_StkSystemCoreClock); ++i)
1022 {
1023 s_StkSystemCoreClock[i] = STK_SYSTEM_CORE_CLOCK_FREQUENCY;
1024 }
1025 }
1026
1027 // Re-derive the s_StkRiscvCsuLock spin-lock timeout from the now-configured core
1028 // clock. Redundant across harts (each hart's Context::Initialize() calls this),
1029 // but idempotent and cheap - and must run before any critical section is entered
1030 // for real, since s_StkSpinlockTimeoutIters otherwise still holds its conservative
1031 // pre-init fallback value.
1032 HW_InitSpinlockTimeout();
1033
1034 // mcycle counter must be enabled per-core
1035 #if STK_SUBMICORSECOND_PRECISION_TIMER
1036 HW_EnableCycleCounter();
1037 #endif
1038 }
1039
1040 __stk_forceinline void ProcessTick()
1041 {
1042 // process tick - scheduler may update m_stack_active to point at a new task
1043 Word cs;
1044 HW_CriticalSectionStart(cs);
1045
1046 #if STK_TICKLESS_IDLE
1047 Timeout ticks = m_sleep_ticks;
1048 #endif
1049
1050 if (m_handler->OnTick(m_stack_idle, m_stack_active
1051 #if STK_TICKLESS_IDLE
1052 , ticks
1053 #endif
1054 ))
1055 {
1056 // refresh ISR asm pointer cache so the naked ISR reads the correct
1057 // (possibly new) active stack SP immediately when jal returns
1058 // s_StkRiscvStackActive[hart] always points to Context::m_stack_active, the pointer
1059 // itself is stable, but we reassign here so multi-core hart-indexed builds
1060 // stay correct if the hart mapping ever changes in future,
1061 // for single-core builds this is a simple store to a known address at index 0
1062 const uint8_t hart = HW_GetHartId();
1063 s_StkRiscvStackActive[hart] = m_stack_active;
1064 #ifdef _STK_RISCV_USE_PENDSV
1065 s_StkRiscvStackIdle[hart] = m_stack_idle;
1066 #endif
1067
1068 #if STK_SEGGER_SYSVIEW
1069 if (m_stack_idle->tid != SYS_TASK_ID_SLEEP)
1070 {
1071 // previous task went idle
1072 SEGGER_SYSVIEW_OnTaskStopReady(m_stack_idle->tid, TRACE_EVENT_SWITCH);
1073 }
1074
1075 if (m_stack_active->tid != SYS_TASK_ID_SLEEP)
1076 {
1077 // current task resumed execution
1078 SEGGER_SYSVIEW_OnTaskStartExec(m_stack_active->tid);
1079 }
1080 #endif
1081
1082 #ifdef _STK_RISCV_USE_PENDSV
1083 HW_ScheduleContextSwitch(hart);
1084 #endif
1085 }
1086
1087 #if STK_TICKLESS_IDLE
1088 m_sleep_ticks = ticks;
1089 #endif
1090
1091 HW_CriticalSectionEnd(cs);
1092 }
1093
1094 __stk_forceinline void EnterCriticalSection()
1095 {
1096 // disable local interrupts and save state
1097 Word current_ses;
1098 HW_CriticalSectionStart(current_ses);
1099
1100 if (m_csu_nesting == 0U)
1101 {
1102 // ONLY attempt the global spinlock if we aren't already nested
1103 HW_SpinLockLock(s_StkRiscvCsuLock);
1104
1105 // store the hardware interrupt state to restore later
1106 m_csu = current_ses;
1107 }
1108
1109 // increase nesting count within a limit
1110 if (++m_csu_nesting > STK_CS_NESTINGS_MAX)
1111 {
1112 // invariant violated: exceeded max allowed number of recursions
1113 STK_KERNEL_PANIC(KERNEL_PANIC_CS_NESTING_OVERFLOW);
1114 }
1115 }
1116
1117 __stk_forceinline void ExitCriticalSection()
1118 {
1119 STK_ASSERT(m_csu_nesting != 0U);
1120 --m_csu_nesting;
1121
1122 if (m_csu_nesting == 0U)
1123 {
1124 // capture the state before releasing lock
1125 const Word ses_to_restore = m_csu;
1126
1127 // release global lock
1128 HW_SpinLockUnlock(s_StkRiscvCsuLock);
1129
1130 // restore hardware interrupts
1131 HW_CriticalSectionEnd(ses_to_restore);
1132 }
1133 }
1134
1135 uint64_t GetSleepTicksPrev()
1136 {
1137 #if STK_TICKLESS_IDLE
1138 const uint64_t ticks = (static_cast<uint64_t>(m_sleep_ticks) * static_cast<uint64_t>(m_tick_period));
1139 #else
1140 const uint64_t ticks = (1U * static_cast<uint64_t>(m_tick_period));
1141 #endif
1142 return ticks;
1143 }
1144
1145 uint64_t GetTimeNow(uint64_t &error)
1146 {
1147 const uint64_t mtime_now = HW_GetMtime();
1148 error = (mtime_now - m_last_mtime) - GetSleepTicksPrev();
1149 return mtime_now;
1150 }
1151
1152 void RearmTimer(const uint64_t mtime_now, const uint64_t error)
1153 {
1154 #if STK_TICKLESS_IDLE
1155 // guard against overflow (theoretical at normal tick periods and CPU frequencies)
1156 STK_ASSERT((static_cast<uint64_t>(m_sleep_ticks) * static_cast<uint64_t>(m_tick_period)) <= (UINT64_MAX - mtime_now));
1157 const uint64_t next_time = (static_cast<uint64_t>(m_sleep_ticks) * static_cast<uint64_t>(m_tick_period));
1158 #else
1159 const uint64_t next_time = (1U * static_cast<uint64_t>(m_tick_period));
1160 #endif
1161 HW_SetMtimecmp(mtime_now + next_time - error);
1162 m_last_mtime = mtime_now;
1163 }
1164
1165 __stk_forceinline void OnSwitchContext()
1166 {
1167 // capture mtime at ISR entry as the absolute base for the next period;
1168 // this eliminates drift from time spent inside OnTick regardless of how
1169 // long the scheduler takes to run
1170 uint64_t error = 0U;
1171 const uint64_t mtime_now = GetTimeNow(error);
1172 __stk_compiler_barrier(); // avoid compiler reordering, we count ticks from this point
1173
1174 // make sure timer is enabled by the Kernel::Start(), disable its start anywhere else
1175 STK_ASSERT(m_started);
1176 STK_ASSERT(m_handler != nullptr);
1177
1178 // process tick - scheduler may update m_stack_active and m_sleep_ticks
1179 ProcessTick();
1180
1181 // rearm timer: use the ISR-entry mtime snapshot as the absolute base so
1182 // any CPU cycles consumed by OnTick do not accumulate as period drift
1183 RearmTimer(mtime_now, error);
1184 }
1185
1186 __stk_forceinline void OnForceContextSwitch(TId id)
1187 {
1188 if (m_handler->OnForceContextSwitch(id, m_stack_idle, m_stack_active))
1189 {
1190 // refresh ISR asm pointer cache so STK_MSI_HANDLER reads the correct
1191 // (possibly new) active/idle stack SP
1192 //
1193 // note: unlike ProcessTick(), the idle slot is refreshed unconditionally here,
1194 // not just under _STK_RISCV_USE_PENDSV; a forced switch can be requested from
1195 // arbitrary task/ISR context (not necessarily from inside STK_SYSTICK_HANDLER),
1196 // so it can never be applied inline - it always goes through STK_MSI_HANDLER
1197 const uint8_t hart = HW_GetHartId();
1198 s_StkRiscvStackActive[hart] = m_stack_active;
1199 s_StkRiscvStackIdle[hart] = m_stack_idle;
1200
1201 // always pend the MSI, regardless of _STK_RISCV_USE_PENDSV
1202 HW_ScheduleContextSwitch(hart);
1203 }
1204 }
1205
1206 void StartTickTimer(Timeout elapsed_ticks)
1207 {
1208 #if STK_TICKLESS_IDLE
1209 // reset sleep ticks if kernel was restarted
1210 m_sleep_ticks = elapsed_ticks;
1211 #else
1212 STK_UNUSED(elapsed_ticks);
1213 #endif
1214
1215 // start timer with default periodicity
1216 m_last_mtime = HW_GetMtime();
1217 HW_SetMtimecmp(m_last_mtime + m_tick_period);
1218
1219 // enable timer interrupt
1220 set_csr(mie, MIP_MTIP);
1221 }
1222
1223 void OnSleepOverride()
1224 {
1225 #if STK_TICKLESS_IDLE
1226 const Timeout sleep_ticks = m_sleep_ticks;
1227 #else
1228 const Timeout sleep_ticks = 1;
1229 #endif
1230
1231 if (!m_overrider->OnSleep(sleep_ticks))
1232 {
1233 HW_EnterSleepMode();
1234 }
1235 }
1236
1237#if STK_TLS && !STK_INLINE_TLS
1238 Word GetTls()
1239 {
1240 hw::CriticalSection::ScopedLock cs_;
1241
1242 STK_ASSERT(m_stack_active != nullptr);
1243
1244 return m_stack_active->tls;
1245 }
1246
1247 void SetTls(Word tp)
1248 {
1249 hw::CriticalSection::ScopedLock cs_;
1250
1251 STK_ASSERT(m_stack_active != nullptr);
1252
1253 m_stack_active->tls = tp;
1254 }
1255#endif // STK_TLS && !STK_INLINE_TLS
1256
1257 void Start();
1258 void OnStart();
1259 void OnStop();
1260#if STK_TICKLESS_IDLE
1261 Timeout Suspend();
1262 void Resume(Timeout elapsed_ticks);
1263#endif
1264
1265 typedef IPlatform::IEventOverrider eovrd_t;
1266 typedef PlatformRiscV::ISpecificEventHandler sehndl_t;
1267 typedef StackMemoryWrapper<STK_RISCV_ISR_STACK_SIZE>::MemoryType isrmem_t;
1268
1269 Stack m_stack_main;
1270 Stack m_stack_isr;
1271 isrmem_t m_stack_isr_mem;
1272 drv::JmpFrame m_exit_buf;
1273 eovrd_t *m_overrider;
1274 sehndl_t *m_specific;
1275 uint32_t m_tick_period;
1276 uint64_t m_last_mtime;
1277#if STK_TICKLESS_IDLE
1278 Timeout m_sleep_ticks;
1279#endif
1280 Word m_csu;
1281 uint8_t m_csu_nesting;
1282 bool m_starting;
1283 bool m_started;
1284 volatile bool m_exiting;
1285};
1286
1291Context s_StkPlatformContext[STK_ARCH_CPU_COUNT];
1292
1293// ----------------------------------------------------------------------------
1294} // namespace
1295// ----------------------------------------------------------------------------
1296
1297__stk_attr_noinline // keep out of inlining to preserve stack frame
1298__stk_attr_noreturn // never returns - a trap
1300{
1301 g_StkLastPanicId = id;
1302
1303 // disable all maskable interrupts: this prevents scheduler from running again and corrupting state further
1304 HW_DisableInterrupts();
1305
1306 // spin forever: with a watchdog active this produces a clean reset, without a watchdog,
1307 // a debugger can attach and inspect 'id'
1308 for (;;)
1309 {
1310 __stk_relax_cpu();
1311 }
1312}
1313
1315{
1316#ifdef _STK_RISCV_USE_PENDSV
1317 Word cs;
1318 HW_CriticalSectionStart(cs);
1319
1320 GetContext().ProcessTick();
1321
1322 HW_CriticalSectionEnd(cs);
1323#else
1324 // unsupported scenario
1325 STK_ASSERT(false);
1326#endif
1327}
1328
1329#define STK_ASM_SAVE_CONTEXT_BASE\
1330 SREG " x1, 2*" REGBYTES "(sp) \n"\
1331 /*SREG " x2, 3*" REGBYTES "(sp) \n" // skip saving sp, Stack pointer */\
1332 /*SREG " x3, 4*" REGBYTES "(sp) \n" // skip saving gp, Global pointer (note: slot is used by fscsr) */\
1333 SREG " x4, 5*" REGBYTES "(sp) \n"\
1334 SREG " x5, 6*" REGBYTES "(sp) \n"\
1335 SREG " x6, 7*" REGBYTES "(sp) \n"\
1336 SREG " x7, 8*" REGBYTES "(sp) \n"\
1337 SREG " x8, 9*" REGBYTES "(sp) \n"\
1338 SREG " x9, 10*" REGBYTES "(sp) \n"\
1339 SREG " x10, 11*" REGBYTES "(sp) \n"\
1340 SREG " x11, 12*" REGBYTES "(sp) \n"\
1341 SREG " x12, 13*" REGBYTES "(sp) \n"\
1342 SREG " x13, 14*" REGBYTES "(sp) \n"\
1343 SREG " x14, 15*" REGBYTES "(sp) \n"\
1344 SREG " x15, 16*" REGBYTES "(sp) \n"
1345
1346#if (__riscv_32e != 1)
1347#define STK_ASM_SAVE_CONTEXT_RV32I_EXT\
1348 SREG " x16, 17*" REGBYTES "(sp) \n"\
1349 SREG " x17, 18*" REGBYTES "(sp) \n"\
1350 SREG " x18, 19*" REGBYTES "(sp) \n"\
1351 SREG " x19, 20*" REGBYTES "(sp) \n"\
1352 SREG " x20, 21*" REGBYTES "(sp) \n"\
1353 SREG " x21, 22*" REGBYTES "(sp) \n"\
1354 SREG " x22, 23*" REGBYTES "(sp) \n"\
1355 SREG " x23, 24*" REGBYTES "(sp) \n"\
1356 SREG " x24, 25*" REGBYTES "(sp) \n"\
1357 SREG " x25, 26*" REGBYTES "(sp) \n"\
1358 SREG " x26, 27*" REGBYTES "(sp) \n"\
1359 SREG " x27, 28*" REGBYTES "(sp) \n"\
1360 SREG " x28, 29*" REGBYTES "(sp) \n"\
1361 SREG " x29, 30*" REGBYTES "(sp) \n"\
1362 SREG " x30, 31*" REGBYTES "(sp) \n"\
1363 SREG " x31, 32*" REGBYTES "(sp) \n"
1364#else
1365#define STK_ASM_SAVE_CONTEXT_RV32I_EXT
1366#endif
1367
1368#if (STK_RISCV_FPU != 0)
1369#define STK_ASM_SAVE_CONTEXT_FP\
1370 FSREG " f0, " FOFFSET "+0*" FREGBYTES "(sp) \n"\
1371 FSREG " f1, " FOFFSET "+1*" FREGBYTES "(sp) \n"\
1372 FSREG " f2, " FOFFSET "+2*" FREGBYTES "(sp) \n"\
1373 FSREG " f3, " FOFFSET "+3*" FREGBYTES "(sp) \n"\
1374 FSREG " f4, " FOFFSET "+4*" FREGBYTES "(sp) \n"\
1375 FSREG " f5, " FOFFSET "+5*" FREGBYTES "(sp) \n"\
1376 FSREG " f6, " FOFFSET "+6*" FREGBYTES "(sp) \n"\
1377 FSREG " f7, " FOFFSET "+7*" FREGBYTES "(sp) \n"\
1378 FSREG " f8, " FOFFSET "+8*" FREGBYTES "(sp) \n"\
1379 FSREG " f9, " FOFFSET "+9*" FREGBYTES "(sp) \n"\
1380 FSREG " f10, " FOFFSET "+10*" FREGBYTES "(sp) \n"\
1381 FSREG " f11, " FOFFSET "+11*" FREGBYTES "(sp) \n"\
1382 FSREG " f12, " FOFFSET "+12*" FREGBYTES "(sp) \n"\
1383 FSREG " f13, " FOFFSET "+13*" FREGBYTES "(sp) \n"\
1384 FSREG " f14, " FOFFSET "+14*" FREGBYTES "(sp) \n"\
1385 FSREG " f15, " FOFFSET "+15*" FREGBYTES "(sp) \n"\
1386 FSREG " f16, " FOFFSET "+16*" FREGBYTES "(sp) \n"\
1387 FSREG " f17, " FOFFSET "+17*" FREGBYTES "(sp) \n"\
1388 FSREG " f18, " FOFFSET "+18*" FREGBYTES "(sp) \n"\
1389 FSREG " f19, " FOFFSET "+19*" FREGBYTES "(sp) \n"\
1390 FSREG " f20, " FOFFSET "+20*" FREGBYTES "(sp) \n"\
1391 FSREG " f21, " FOFFSET "+21*" FREGBYTES "(sp) \n"\
1392 FSREG " f22, " FOFFSET "+22*" FREGBYTES "(sp) \n"\
1393 FSREG " f23, " FOFFSET "+23*" FREGBYTES "(sp) \n"\
1394 FSREG " f24, " FOFFSET "+24*" FREGBYTES "(sp) \n"\
1395 FSREG " f25, " FOFFSET "+25*" FREGBYTES "(sp) \n"\
1396 FSREG " f26, " FOFFSET "+26*" FREGBYTES "(sp) \n"\
1397 FSREG " f27, " FOFFSET "+27*" FREGBYTES "(sp) \n"\
1398 FSREG " f28, " FOFFSET "+28*" FREGBYTES "(sp) \n"\
1399 FSREG " f29, " FOFFSET "+29*" FREGBYTES "(sp) \n"\
1400 FSREG " f30, " FOFFSET "+30*" FREGBYTES "(sp) \n"\
1401 FSREG " f31, " FOFFSET "+31*" FREGBYTES "(sp) \n"
1402#else
1403#define STK_ASM_SAVE_CONTEXT_FP
1404#endif
1405
1406#define STK_ASM_SAVE_CONTEXT_PC_STATUS\
1407 "csrr t0, mepc \n"\
1408 "csrr t1, mstatus \n"\
1409 SREG " t0, 0*" REGBYTES "(sp) \n"\
1410 SREG " t1, 1*" REGBYTES "(sp) \n"
1411
1412#if (STK_RISCV_FPU != 0)
1413#define STK_ASM_SAVE_CONTEXT_FRCSR\
1414 "frcsr t0 \n"\
1415 SREG " t0, 4*" REGBYTES "(sp) \n" /* use stack memory slot of gp (see comment for x3 above) */
1416#else
1417#define STK_ASM_SAVE_CONTEXT_FRCSR
1418#endif
1419
1420#define STK_ASM_SAVE_CONTEXT\
1421 "addi sp, sp, -" REGSIZE " \n" /* allocate stack memory for registers */\
1422 STK_ASM_SAVE_CONTEXT_BASE\
1423 STK_ASM_SAVE_CONTEXT_RV32I_EXT\
1424 STK_ASM_SAVE_CONTEXT_FP\
1425 STK_ASM_SAVE_CONTEXT_PC_STATUS\
1426 STK_ASM_SAVE_CONTEXT_FRCSR
1427
1428#define STK_ASM_LOAD_CONTEXT_BASE\
1429 LREG " x1, 2*" REGBYTES "(sp) \n"\
1430 /*LREG " x2, 3*" REGBYTES "(sp) \n" skip loading sp, Stack pointer */\
1431 /*LREG " x3, 4*" REGBYTES "(sp) \n" skip loading gp, Global pointer (note: slot is used by fscsr) */\
1432 LREG " x4, 5*" REGBYTES "(sp) \n"\
1433 LREG " x5, 6*" REGBYTES "(sp) \n"\
1434 LREG " x6, 7*" REGBYTES "(sp) \n"\
1435 LREG " x7, 8*" REGBYTES "(sp) \n"\
1436 LREG " x8, 9*" REGBYTES "(sp) \n"\
1437 LREG " x9, 10*" REGBYTES "(sp) \n"\
1438 LREG " x10, 11*" REGBYTES "(sp) \n"\
1439 LREG " x11, 12*" REGBYTES "(sp) \n"\
1440 LREG " x12, 13*" REGBYTES "(sp) \n"\
1441 LREG " x13, 14*" REGBYTES "(sp) \n"\
1442 LREG " x14, 15*" REGBYTES "(sp) \n"\
1443 LREG " x15, 16*" REGBYTES "(sp) \n"
1444
1445#if (__riscv_32e != 1)
1446#define STK_ASM_LOAD_CONTEXT_RV32I_EXT\
1447 LREG " x16, 17*" REGBYTES "(sp) \n"\
1448 LREG " x17, 18*" REGBYTES "(sp) \n"\
1449 LREG " x18, 19*" REGBYTES "(sp) \n"\
1450 LREG " x19, 20*" REGBYTES "(sp) \n"\
1451 LREG " x20, 21*" REGBYTES "(sp) \n"\
1452 LREG " x21, 22*" REGBYTES "(sp) \n"\
1453 LREG " x22, 23*" REGBYTES "(sp) \n"\
1454 LREG " x23, 24*" REGBYTES "(sp) \n"\
1455 LREG " x24, 25*" REGBYTES "(sp) \n"\
1456 LREG " x25, 26*" REGBYTES "(sp) \n"\
1457 LREG " x26, 27*" REGBYTES "(sp) \n"\
1458 LREG " x27, 28*" REGBYTES "(sp) \n"\
1459 LREG " x28, 29*" REGBYTES "(sp) \n"\
1460 LREG " x29, 30*" REGBYTES "(sp) \n"\
1461 LREG " x30, 31*" REGBYTES "(sp) \n"\
1462 LREG " x31, 32*" REGBYTES "(sp) \n"
1463#else
1464#define STK_ASM_LOAD_CONTEXT_RV32I_EXT
1465#endif
1466
1467#if (STK_RISCV_FPU != 0)
1468#define STK_ASM_LOAD_CONTEXT_FP\
1469 FLREG " f0, " FOFFSET "+0*" FREGBYTES "(sp) \n"\
1470 FLREG " f1, " FOFFSET "+1*" FREGBYTES "(sp) \n"\
1471 FLREG " f2, " FOFFSET "+2*" FREGBYTES "(sp) \n"\
1472 FLREG " f3, " FOFFSET "+3*" FREGBYTES "(sp) \n"\
1473 FLREG " f4, " FOFFSET "+4*" FREGBYTES "(sp) \n"\
1474 FLREG " f5, " FOFFSET "+5*" FREGBYTES "(sp) \n"\
1475 FLREG " f6, " FOFFSET "+6*" FREGBYTES "(sp) \n"\
1476 FLREG " f7, " FOFFSET "+7*" FREGBYTES "(sp) \n"\
1477 FLREG " f8, " FOFFSET "+8*" FREGBYTES "(sp) \n"\
1478 FLREG " f9, " FOFFSET "+9*" FREGBYTES "(sp) \n"\
1479 FLREG " f10, " FOFFSET "+10*" FREGBYTES "(sp) \n"\
1480 FLREG " f11, " FOFFSET "+11*" FREGBYTES "(sp) \n"\
1481 FLREG " f12, " FOFFSET "+12*" FREGBYTES "(sp) \n"\
1482 FLREG " f13, " FOFFSET "+13*" FREGBYTES "(sp) \n"\
1483 FLREG " f14, " FOFFSET "+14*" FREGBYTES "(sp) \n"\
1484 FLREG " f15, " FOFFSET "+15*" FREGBYTES "(sp) \n"\
1485 FLREG " f16, " FOFFSET "+16*" FREGBYTES "(sp) \n"\
1486 FLREG " f17, " FOFFSET "+17*" FREGBYTES "(sp) \n"\
1487 FLREG " f18, " FOFFSET "+18*" FREGBYTES "(sp) \n"\
1488 FLREG " f19, " FOFFSET "+19*" FREGBYTES "(sp) \n"\
1489 FLREG " f20, " FOFFSET "+20*" FREGBYTES "(sp) \n"\
1490 FLREG " f21, " FOFFSET "+21*" FREGBYTES "(sp) \n"\
1491 FLREG " f22, " FOFFSET "+22*" FREGBYTES "(sp) \n"\
1492 FLREG " f23, " FOFFSET "+23*" FREGBYTES "(sp) \n"\
1493 FLREG " f24, " FOFFSET "+24*" FREGBYTES "(sp) \n"\
1494 FLREG " f25, " FOFFSET "+25*" FREGBYTES "(sp) \n"\
1495 FLREG " f26, " FOFFSET "+26*" FREGBYTES "(sp) \n"\
1496 FLREG " f27, " FOFFSET "+27*" FREGBYTES "(sp) \n"\
1497 FLREG " f28, " FOFFSET "+28*" FREGBYTES "(sp) \n"\
1498 FLREG " f29, " FOFFSET "+29*" FREGBYTES "(sp) \n"\
1499 FLREG " f30, " FOFFSET "+30*" FREGBYTES "(sp) \n"\
1500 FLREG " f31, " FOFFSET "+31*" FREGBYTES "(sp) \n"
1501#else
1502#define STK_ASM_LOAD_CONTEXT_FP
1503#endif
1504
1505#define STK_ASM_LOAD_CONTEXT_PC_STATUS\
1506 LREG " t0, 0*" REGBYTES "(sp) \n"\
1507 LREG " t1, 1*" REGBYTES "(sp) \n"\
1508 "csrw mepc, t0 \n"\
1509 "csrw mstatus, t1 \n"
1510
1511#if (STK_RISCV_FPU != 0)
1512#define STK_ASM_LOAD_CONTEXT_FRCSR\
1513 LREG " t0, 4*" REGBYTES "(sp) \n" /* use stack memory slot of gp (see comment for x3 below) */\
1514 "fscsr t0 \n"
1515#else
1516#define STK_ASM_LOAD_CONTEXT_FRCSR
1517#endif
1518
1519#define STK_ASM_LOAD_CONTEXT\
1520 STK_ASM_LOAD_CONTEXT_PC_STATUS\
1521 STK_ASM_LOAD_CONTEXT_FRCSR\
1522 STK_ASM_LOAD_CONTEXT_BASE\
1523 STK_ASM_LOAD_CONTEXT_RV32I_EXT\
1524 STK_ASM_LOAD_CONTEXT_FP\
1525 "addi sp, sp, " REGSIZE " \n" /* shrink stack memory of registers */
1526
1527// ----------------------------------------------------------------------------
1528// Make file local content (anonymous namespace): helpers which depend on Context.
1529namespace {
1530// ----------------------------------------------------------------------------
1531
1532__stk_forceinline void HW_LoadContextAndExit()
1533{
1534 __asm volatile(
1535 LREG " t0, %0 \n" // load the first member (SP) into t0
1536 LREG " sp, 0(t0) \n" // sp = t0
1537
1538 STK_ASM_LOAD_CONTEXT
1539 STK_ASM_EXIT_FROM_HANDLER " \n"
1540
1541 : /* output: none */
1542 : "m"(GetContext().m_stack_active)
1543 : "t0", "t1", "a2", "a3", "a4", "a5", "gp", "memory");
1544}
1545
1546__stk_forceinline void HW_EnableFullFpuAccess()
1547{
1548#if (STK_RISCV_FPU != 0)
1549 __asm volatile(
1550 "csrs mstatus, %0"
1551 : /* output: none */
1552 : "r"(MSTATUS_FS | MSTATUS_XS)
1553 : "memory" /* ensure no FP instructions are moved before this call */);
1554#endif
1555}
1556
1557__stk_forceinline void HW_ClearFpuState()
1558{
1559#if (STK_RISCV_FPU != 0)
1560 __asm volatile(
1561 "fssr x0"
1562 : /* output: none */
1563 : /* input: none */
1564 : "memory" /* ensure flags are cleared before next FP op */);
1565#endif
1566}
1567
1568__stk_forceinline void HW_SaveMainSP()
1569{
1570 __asm volatile(
1571 SREG " sp, %0"
1572 : "=m"(GetContext().m_stack_main)
1573 : /* input: none */
1574 : "memory" /* protect against compiler reordering */ );
1575}
1576
1577__stk_forceinline void HW_LoadMainSP()
1578{
1579 __asm volatile(
1580 LREG " sp, %0"
1581 : /* output: none */
1582 : "m"(GetContext().m_stack_main)
1583 : "memory" /* protect against compiler reordering */ );
1584}
1585
1595__stk_forceinline Word HW_GetCurrentException()
1596{
1597 Word mcause;
1598 __asm volatile("csrr %0, mcause" : "=r"(mcause));
1599 return mcause;
1600}
1601
1602__stk_forceinline bool HW_IsHandlerMode()
1603{
1604 const Word current_sp = HW_GetCallerSP();
1605
1606 // get the bounds of the ISR stack from our Context
1607 // note: STK uses StackMemoryWrapper, so we check against that memory block
1608 const Word isr_stack_base = hw::PtrToWord(&GetContext().m_stack_isr_mem);
1609 const Word isr_stack_top = isr_stack_base + (STK_RISCV_ISR_STACK_SIZE * sizeof(Word));
1610
1611 return ((current_sp >= isr_stack_base) && (current_sp < isr_stack_top));
1612}
1613
1614__stk_forceinline void OnTaskStart()
1615{
1616 HW_LoadContextAndExit();
1617}
1618
1619// ----------------------------------------------------------------------------
1620} // namespace
1621// ----------------------------------------------------------------------------
1622
1623// __stk_attr_used for LTO
1624extern "C" STK_RISCV_ISR_SECTION __stk_attr_used void TrySwitchContext()
1625{
1626 GetContext().OnSwitchContext();
1627}
1628
1629#ifdef _STK_RISCV_USE_PENDSV
1630extern "C" STK_RISCV_ISR_SECTION __stk_attr_naked void STK_SYSTICK_HANDLER()
1631{
1632 __asm volatile(
1633 // 1. save full interrupted context onto the task stack
1634 STK_ASM_SAVE_CONTEXT
1635
1636 // 2. store task SP into s_StkRiscvSpIsrInt[hart] directly (plain Word, no struct indirection)
1637#if (STK_ARCH_CPU_COUNT > 1)
1638 "csrr t0, mhartid \n"
1639 "la t1, s_StkRiscvSpIsrInt \n"
1640 "slli t0, t0, " REGBYTES_LOG2 " \n" // t0 = hart * sizeof(Word)
1641 "add t1, t1, t0 \n" // t1 = &s_StkRiscvSpIsrInt[hart]
1642 SREG " sp, 0(t1) \n" // store sp directly - no pointer dereference
1643#else
1644 "la t1, s_StkRiscvSpIsrInt \n"
1645 SREG " sp, 0(t1) \n" // store sp directly - no pointer dereference
1646#endif
1647
1648 // 3. switch to private ISR stack
1649#if (STK_ARCH_CPU_COUNT > 1)
1650 "csrr t0, mhartid \n"
1651 "la t1, s_StkRiscvStackIsr \n"
1652 "slli t0, t0, " REGBYTES_LOG2 " \n"
1653 "add t1, t1, t0 \n"
1654 LREG " t1, 0(t1) \n"
1655#else
1656 "la t1, s_StkRiscvStackIsr \n"
1657 LREG " t1, 0(t1) \n"
1658#endif
1659 LREG " sp, 0(t1) \n" // sp = Stack::SP of ISR stack
1660
1661 // 4. run scheduler
1662 "jal ra, TrySwitchContext \n"
1663
1664 // 5. restore the interrupted task's SP from s_StkRiscvSpIsrInt[hart]
1665#if (STK_ARCH_CPU_COUNT > 1)
1666 "csrr t0, mhartid \n"
1667 "la t1, s_StkRiscvSpIsrInt \n"
1668 "slli t0, t0, " REGBYTES_LOG2 " \n"
1669 "add t1, t1, t0 \n"
1670 LREG " sp, 0(t1) \n" // sp = saved task SP - direct load, no struct
1671#else
1672 "la t1, s_StkRiscvSpIsrInt \n"
1673 LREG " sp, 0(t1) \n" // sp = saved task SP - direct load, no struct
1674#endif
1675
1676 // 6. restore context
1677 STK_ASM_LOAD_CONTEXT
1678
1679 // 7. exit ISR handler
1680 STK_ASM_EXIT_FROM_HANDLER " \n"
1681
1682 : /* outputs: none - naked, compiler emits nothing outside this asm */
1683 : /* inputs: all addresses loaded as linker symbols via "la" */
1684 : /* clobbers: none - the asm string owns all registers */);
1685}
1686#endif // _STK_RISCV_USE_PENDSV
1687
1709extern "C" STK_RISCV_ISR_SECTION __stk_attr_naked void STK_MSI_HANDLER()
1710{
1711 __asm volatile(
1712 // 1. save context
1713 STK_ASM_SAVE_CONTEXT
1714
1715 // 2. store task SP into s_StkRiscvStackIdle[hart]->SP
1716 // all integer registers are now saved. t0/t1 are free to use as scratch.
1717 // "la" loads the address of the global array - a linker-time constant,
1718 // no compiler-generated runtime code, safe to use here
1719#if (STK_ARCH_CPU_COUNT > 1)
1720 "csrr t0, mhartid \n"
1721 "la t1, s_StkRiscvStackIdle \n"
1722 "slli t0, t0, " REGBYTES_LOG2 " \n" // t0 = hart * sizeof(Stack*)
1723 "add t1, t1, t0 \n" // t1 = &s_StkRiscvStackIdle[hart]
1724 LREG " t1, 0(t1) \n" // t1 = s_StkRiscvStackIdle[hart] (Stack*)
1725#else
1726 "la t1, s_StkRiscvStackIdle \n"
1727 LREG " t1, 0(t1) \n" // t1 = s_StkRiscvStackIdle[0] (Stack*)
1728#endif
1729 SREG " sp, 0(t1) \n" // Stack::SP = task's sp (SP is first member)
1730
1731 // 3. clear exception: MSIP[hart] = 0
1732#if (STK_ARCH_CPU_COUNT > 1)
1733 "csrr t0, mhartid \n"
1734 "slli t0, t0, 2 \n" // t0 = hart * 4
1735 "li t1, %[clint_msip_base] \n"
1736 "add t0, t0, t1 \n" // t0 = &MSIP[hart]
1737#else
1738 "li t0, %[clint_msip_base] \n" // t0 = &MSIP[0]
1739#endif
1740 "sw zero, 0(t0) \n" // MSIP[hart] = 0
1741 "fence rw, rw \n" // fence rw,rw - ensure the write is visible before re-enable
1742
1743 // 4. load SP from s_StkRiscvStackActive[hart]->SP
1744#if (STK_ARCH_CPU_COUNT > 1)
1745 "csrr t0, mhartid \n"
1746 "la t1, s_StkRiscvStackActive \n"
1747 "slli t0, t0, " REGBYTES_LOG2 " \n"
1748 "add t1, t1, t0 \n"
1749 LREG " t1, 0(t1) \n"
1750#else
1751 "la t1, s_StkRiscvStackActive \n"
1752 LREG " t1, 0(t1) \n"
1753#endif
1754 LREG " sp, 0(t1) \n" // sp = active task's saved SP
1755
1756 // 5. load context of the active task
1757 STK_ASM_LOAD_CONTEXT
1758
1759 // 6. exit ISR handler
1760 STK_ASM_EXIT_FROM_HANDLER " \n"
1761
1762 : /* outputs: none - naked, compiler emits nothing outside this asm */
1763 : [clint_msip_base] "i" (STK_RISCV_CLINT_BASE_ADDR) /* other inputs: all addresses loaded as linker symbols via "la" */
1764 : /* clobbers: none - the asm string owns all registers */);
1765}
1766
1767#ifndef _STK_RISCV_USE_PENDSV
1768/* STK_SYSTICK_HANDLER
1769
1770RISC-V machine-timer ISR: Saves the interrupted task's full context, switches
1771to the private ISR stack, calls TrySwitchContext (which reschedules the timer
1772and runs the scheduler), then restores the (possibly new) task's context.
1773
1774DESIGN RULES - must be obeyed to work correctly at all optimisation levels:
1775
1776 1. Single asm volatile, no compiler operands.
1777 The function body is ONE __asm volatile("..." : : : ) with empty
1778 input/output/clobber lists. No "m" or "r" constraints are used because
1779 the compiler evaluates those as C expressions BEFORE emitting any asm
1780 text, i.e. before the register save - trashing uninitialized registers.
1781
1782 2. All addresses are linker symbols loaded via "la" inside the asm.
1783 s_StkRiscvStackActive and s_StkRiscvStackIsr are plain file-scope globals
1784 with C linkage (declared extern "C" above). "la reg, sym" emits a PC-relative
1785 load that is resolved at link time, it produces no compiler-generated code
1786 outside the asm string.
1787
1788 3. Stack pointer indexing uses sizeof(Stack*) == REGBYTES.
1789 For multi-hart builds the array index is hart * REGBYTES, which is a
1790 single left-shift by log2(REGBYTES): 2 for RV32 (4 bytes), 3 for RV64
1791 (8 bytes). REGBYTES_LOG2 is defined below accordingly.
1792
1793 4. s_StkRiscvStackActive[hart]->SP is updated by TrySwitchContext.
1794 The naked asm reads it fresh after the jal returns, so it always sees
1795 the task the scheduler has chosen - even if it changed.
1796
1797 Stack frame layout (offsets from sp after "addi sp,-REGSIZE"):
1798 [0*REGBYTES] mepc (service slot 0)
1799 [1*REGBYTES] mstatus (service slot 1)
1800 [2*REGBYTES] x1 / ra
1801 [3*REGBYTES] x2 / sp - SKIPPED, managed explicitly
1802 [4*REGBYTES] x3 / gp - SKIPPED, fixed register; slot reused for FCSR
1803 [5*REGBYTES] x4 / tp
1804 [6*REGBYTES] x5 / t0
1805 ...
1806 [32*REGBYTES] x31 / t6 (RV32I; absent on RV32E)
1807 [FOFFSET + n*FREGBYTES] fn (FP registers, if STK_RISCV_FPU != 0)
1808*/
1809extern "C" STK_RISCV_ISR_SECTION __stk_attr_naked void STK_SYSTICK_HANDLER()
1810{
1811 __asm volatile(
1812 // 1. save context
1813 STK_ASM_SAVE_CONTEXT
1814
1815 // 2. store task SP into s_StkRiscvStackActive[hart]->SP
1816 // all integer registers are now saved. t0/t1 are free to use as scratch.
1817 // "la" loads the address of the global array - a linker-time constant,
1818 // no compiler-generated runtime code, safe to use here
1819#if (STK_ARCH_CPU_COUNT > 1)
1820 "csrr t0, mhartid \n"
1821 "la t1, s_StkRiscvStackActive \n"
1822 "slli t0, t0, " REGBYTES_LOG2 " \n" // t0 = hart * sizeof(Stack*)
1823 "add t1, t1, t0 \n" // t1 = &s_StkRiscvStackActive[hart]
1824 LREG " t1, 0(t1) \n" // t1 = s_StkRiscvStackActive[hart] (Stack*)
1825#else
1826 "la t1, s_StkRiscvStackActive \n"
1827 LREG " t1, 0(t1) \n" // t1 = s_StkRiscvStackActive[0] (Stack*)
1828#endif
1829 SREG " sp, 0(t1) \n" // Stack::SP = task's sp (SP is first member)
1830
1831 // 3. switch to private ISR stack
1832#if (STK_ARCH_CPU_COUNT > 1)
1833 "csrr t0, mhartid \n"
1834 "la t1, s_StkRiscvStackIsr \n"
1835 "slli t0, t0, " REGBYTES_LOG2 " \n"
1836 "add t1, t1, t0 \n"
1837 LREG " t1, 0(t1) \n" // t1 = s_StkRiscvStackIsr[hart] (Stack*)
1838#else
1839 "la t1, s_StkRiscvStackIsr \n"
1840 LREG " t1, 0(t1) \n" // t1 = s_StkRiscvStackIsr[0] (Stack*)
1841#endif
1842 LREG " sp, 0(t1) \n" // sp = Stack::SP of ISR stack
1843
1844 // 4. call TrySwitchContext
1845 // runs on the ISR stack: reschedules timer, runs scheduler
1846 // (which may update m_stack_active to a new task), then updates
1847 // s_StkRiscvStackActive[hart] so step 5 below reads the correct new SP,
1848 // all caller-saved registers (a0-a7, t0-t6, ra) are trashed - expected
1849 "jal ra, TrySwitchContext \n"
1850
1851 // 5. reload SP from s_StkRiscvStackActive[hart]->SP
1852 // TrySwitchContext updated s_StkRiscvStackActive[hart] before returning,
1853 // we re-read it fresh to pick up any task switch the scheduler made
1854#if (STK_ARCH_CPU_COUNT > 1)
1855 "csrr t0, mhartid \n"
1856 "la t1, s_StkRiscvStackActive \n"
1857 "slli t0, t0, " REGBYTES_LOG2 " \n"
1858 "add t1, t1, t0 \n"
1859 LREG " t1, 0(t1) \n"
1860#else
1861 "la t1, s_StkRiscvStackActive \n"
1862 LREG " t1, 0(t1) \n"
1863#endif
1864 LREG " sp, 0(t1) \n" // sp = active task's saved SP
1865
1866 // 6. load context of the active task
1867 STK_ASM_LOAD_CONTEXT
1868
1869 // 7. exit ISR handler
1870 STK_ASM_EXIT_FROM_HANDLER " \n"
1871
1872 : /* outputs: none - naked, compiler emits nothing outside this asm */
1873 : /* inputs: none - all addresses loaded as linker symbols via "la" */
1874 : /* clobbers: none - the asm string owns all registers */
1875 );
1876}
1877#endif // !_STK_RISCV_USE_PENDSV
1878
1879void Context::OnStart()
1880{
1881 const uint8_t hart = HW_GetHartId();
1882
1883 // save SP of main stack to reuse it for scheduler exit
1884 HW_SaveMainSP();
1885
1886 // enable FPU (if available)
1887 HW_EnableFullFpuAccess();
1888
1889 // clear FPU usage status if FPU was used before kernel start
1890 HW_ClearFpuState();
1891
1892 // notify kernel
1893 m_handler->OnStart(m_stack_active);
1894
1895 // initialize ISR asm pointer cache
1896 s_StkRiscvStackIsr[hart] = &m_stack_isr; // set once here, the ISR stack never moves
1897 s_StkRiscvStackActive[hart] = m_stack_active;
1898 s_StkRiscvStackIdle[hart] = m_stack_idle;
1899
1900 // start with initially 1 elapsed tick (after timer expires)
1901 StartTickTimer(1);
1902
1903 // change state before enabling interrupts
1904 m_started = true;
1905 m_starting = false;
1906
1907 // start recording task execution start
1908#if STK_SEGGER_SYSVIEW
1909 SEGGER_SYSVIEW_OnTaskStartExec(m_stack_active->tid);
1910#endif
1911
1912 // enable SV exception (MSI)
1913 // note: unconditional - even the single-handler (non-PendSV) tick path needs this,
1914 // since Context::OnForceContextSwitch() always triggers a switch via STK_MSI_HANDLER
1915 set_csr(mie, MIP_MSIP);
1916}
1917
1918STK_RISCV_ISR void STK_SVC_HANDLER()
1919{
1920 const Word cause = HW_GetCurrentException();
1921 Context &ctx = GetContext();
1922
1923 /*if (cause & (1UL << (__riscv_xlen - 1)))
1924 {
1925 cause &= ~(1UL << (__riscv_xlen - 1));
1926
1927 if (cause == IRQ_M_TIMER)
1928 {
1929
1930 }
1931 }*/
1932
1933 if (cause == IRQ_M_EXT)
1934 {
1935 // not starting scheduler, then try to forward ecall to user
1936 if (!ctx.m_starting)
1937 {
1938 // forward event to user
1939 if (ctx.m_specific != nullptr)
1940 {
1941 ctx.m_specific->OnException(cause);
1942 }
1943
1944 // switch to the next instruction of the caller space (PC) after the return
1945 write_csr(mepc, read_csr(mepc) + sizeof(Word));
1946 }
1947 else
1948 {
1949 // make sure interrupts do not interfere
1950 HW_DisableInterrupts();
1951
1952 // configure scheduling
1953 ctx.OnStart();
1954
1955 // start first task
1956 OnTaskStart();
1957 }
1958 }
1959 else
1960 {
1961 if (ctx.m_specific != nullptr)
1962 {
1963 // forward event to user
1964 ctx.m_specific->OnException(cause);
1965 }
1966 else
1967 {
1968 // trap further execution
1969 // note: normally, if trapped here with cause 2 or 4 then check stack memory size of the
1970 // tasks, scheduler and ISR, they were likely overwritten if your code is 100% correct
1972 }
1973 }
1974}
1975
1976void drv::OnTaskRun(ITask *task)
1977{
1978 task->Run();
1979}
1980
1981void drv::OnTaskExit()
1982{
1983 const Context &ctx = GetContext();
1984
1985 Word cs;
1986 HW_CriticalSectionStart(cs);
1987
1988 ctx.m_handler->OnTaskExit(ctx.m_stack_active);
1989
1990 HW_CriticalSectionEnd(cs);
1991
1992 for (;;)
1993 {
1994 __DSB(); // data barrier
1995 __WFI(); // enter standby mode until time slot expires
1996 }
1997}
1998
1999STK_RISCV_ISR_SECTION
2000void drv::OnSchedulerSleep()
2001{
2002 // if hit here, increase the size of STK_SLEEP_TRAP_STACK_SIZE
2004
2005 for (;;)
2006 {
2007 #if STK_SEGGER_SYSVIEW
2008 SEGGER_SYSVIEW_OnIdle();
2009 #endif
2010
2011 HW_EnterSleepMode();
2012 }
2013}
2014
2015STK_RISCV_ISR_SECTION
2016void drv::OnSchedulerSleepOverride()
2017{
2018 // if hit here, increase the size of STK_SLEEP_TRAP_STACK_SIZE
2020
2021 for (;;)
2022 {
2023 #if STK_SEGGER_SYSVIEW
2024 SEGGER_SYSVIEW_OnIdle();
2025 #endif
2026
2027 GetContext().OnSleepOverride();
2028 }
2029}
2030
2031void drv::OnSchedulerExit()
2032{
2033 // switch to main stack
2034 HW_LoadMainSP();
2035
2036 // jump to the exit from the IKernel::Start()
2037 drv::RestoreJmp(GetContext().m_exit_buf, 0);
2038}
2039
2040void PlatformRiscV::Initialize(IEventHandler *event_handler, IKernelService *service, uint32_t resolution_us, Stack *exit_trap)
2041{
2042 GetContext().Initialize(event_handler, service, exit_trap, resolution_us);
2043}
2044
2045void Context::Start()
2046{
2047 m_exiting = false;
2048
2049 // save jump location of the Exit trap
2050 STK_UNUSED(drv::SaveJmp(m_exit_buf));
2051 if (m_exiting)
2052 {
2053 // notify kernel about a full stop
2054 m_handler->OnStop();
2055 }
2056 else
2057 {
2058 // enable FPU (if available)
2059 HW_EnableFullFpuAccess();
2060
2061 // start
2062 m_starting = true;
2063 HW_StartScheduler();
2064 }
2065}
2066
2067#if STK_TICKLESS_IDLE
2068Timeout Context::Suspend()
2069{
2070 const uint32_t resolution = static_cast<uint32_t>(ConvertTimeUsToClockCycles(HW_CoreClockFrequency(), m_tick_resolution));
2071 if (resolution == 0U)
2072 {
2073 STK_ASSERT(false);
2074 return NO_WAIT;
2075 }
2076
2077 HW_DisableInterrupts();
2078
2079 // stop tick timer
2080 HW_StopMTimer();
2081
2082 // clear pending PendSV exception
2083 HW_ClearPendingSwitch();
2084
2085 // get already elapsed CPU cycles since SysTick ISR invocation up to SysTick timer stop (see above)
2086 // to account for them for a new period value
2087 const uint32_t elapsed = HW_GetMtime() - m_last_mtime;
2088
2089 // get already elapsed ticks since the OnTick and a call to Suspend(), we shall account for this
2090 // period and return only the remainder
2091 const Timeout elapsed_ticks = static_cast<Timeout>(elapsed / resolution);
2092 const Timeout sleep_ticks = Max(m_sleep_ticks - elapsed_ticks, static_cast<Timeout>(0));
2093
2094 // notify core
2095 m_handler->OnSuspend(true);
2096
2097 HW_EnableInterrupts();
2098
2099 return sleep_ticks;
2100}
2101#endif
2102
2103#if STK_TICKLESS_IDLE
2104void Context::Resume(Timeout elapsed_ticks)
2105{
2106 HW_DisableInterrupts();
2107
2108 // notify core
2109 m_handler->OnSuspend(false);
2110
2111 // start with initially elapsed ticks (OnTick will fire with elapsed_ticks + 1)
2112 StartTickTimer(elapsed_ticks + 1);
2113
2114 HW_EnableInterrupts();
2115}
2116#endif
2117
2119{
2120 GetContext().Start();
2121}
2122
2123void PlatformRiscV::InitStack(EStackType stack_type, Stack *stack, IStackMemory *stack_memory, ITask *user_task)
2124{
2125 // TaskFrame must map exactly onto the slot layout consumed by STK_ASM_SAVE_CONTEXT / STK_ASM_LOAD_CONTEXT - no padding allowed
2126 STK_STATIC_ASSERT_DESC(sizeof(TaskFrame) == (STK_RISCV_REGISTER_COUNT + STK_SERVICE_SLOTS) * sizeof(Word),
2127 "TaskFrame size must match REGSIZE: (REGISTER_COUNT + SERVICE_SLOTS) * REGBYTES");
2128
2129 STK_ASSERT(stack_memory->GetStackSize() > (STK_RISCV_REGISTER_COUNT + STK_SERVICE_SLOTS));
2130
2131 // initialize stack memory (fills all slots with STK_STACK_MEMORY_FILLER)
2132 const Word stack_top = PlatformContext::InitStackMemory(stack_memory);
2133
2134 // initialize Stack Pointer (SP): frame sits at the bottom of the register window
2135 stack->SP = stack_top - ((STK_RISCV_REGISTER_COUNT + STK_SERVICE_SLOTS) * sizeof(Word));
2136
2137 // place the task frame at SP directly at the base of the register window
2138 TaskFrame *const task_frame = hw::WordToPtr<TaskFrame>(stack->SP);
2139
2140 // initialize registers for the user task's first start
2141 switch (stack_type)
2142 {
2143 case STACK_USER_TASK: {
2144 task_frame->MEPC = hw::PtrToWord(&drv::OnTaskRun);
2145 task_frame->X1_RA = hw::PtrToWord(&drv::OnTaskExit);
2146 task_frame->X10_A0 = hw::PtrToWord(user_task);
2147 break; }
2148
2149 case STACK_SLEEP_TRAP: {
2150 task_frame->MEPC = hw::PtrToWord(GetContext().m_overrider != nullptr ? &drv::OnSchedulerSleepOverride : &drv::OnSchedulerSleep);
2151 task_frame->X1_RA = STK_STACK_MEMORY_FILLER; // should not attempt to exit
2152 break; }
2153
2154 case STACK_EXIT_TRAP: {
2155 task_frame->MEPC = hw::PtrToWord(&drv::OnSchedulerExit);
2156 task_frame->X1_RA = STK_STACK_MEMORY_FILLER; // should not attempt to exit
2157 break; }
2158
2159 default: {
2161 break; }
2162 }
2163
2164 // mstatus: return to M-mode (MPP), interrupts enabled on mret (MPIE),
2165 // FPU/extension state initial (FS/XS) if FPU present
2166 task_frame->MSTATUS = MSTATUS_MPP | MSTATUS_MPIE | (STK_RISCV_FPU != 0U ? (MSTATUS_FS | MSTATUS_XS) : 0U);
2167
2168#if (STK_RISCV_FPU != 0)
2169 task_frame->X3_FSR = 0U; // FCSR = 0: round-to-nearest, no accrued exception flags
2170#endif
2171}
2172
2173void Context::OnStop()
2174{
2175 // stop timer
2176 HW_StopMTimer();
2177
2178 // clear pending SV exception
2179 HW_ClearPendingSwitch();
2180
2181 m_started = false;
2182 m_exiting = true;
2183
2184 // make sure all assignments are set and executed
2185 __DSB();
2186 __ISB();
2187}
2188
2190{
2191 GetContext().OnStop();
2192
2193 // load context of the Exit trap
2194 HW_DisableInterrupts();
2195 OnTaskStart();
2196}
2197
2198uint32_t PlatformRiscV::GetTickResolution() const
2199{
2200 return GetContext().m_tick_resolution;
2201}
2202
2204{
2205 return static_cast<Cycles>(HW_GetMtime());
2206}
2207
2209{
2210 return HW_MtimeClockFrequency();
2211}
2212
2214{
2215 Word cs;
2216 HW_CriticalSectionStart(cs);
2217
2218 Context &ctx = GetContext();
2219 STK_ASSERT(ctx.m_started);
2220
2221 ctx.OnForceContextSwitch(id);
2222
2223 HW_CriticalSectionEnd(cs);
2224}
2225
2227{
2228 TId result;
2229
2230 if (HW_IsHandlerMode())
2231 {
2232 // to avoid the collision with TID_ISR_N mask, extract and fit into available space:
2233
2234 const Word exc = HW_GetCurrentException();
2235 const Word num = (exc & 0x7FFU);
2236 #if (__riscv_xlen > 32)
2237 const Word interrupt_bit = ((exc & (1ULL << (__riscv_xlen - 1))) ? 0x800U : 0);
2238 #else
2239 const Word interrupt_bit = ((exc & (1U << (__riscv_xlen - 1))) ? 0x800U : 0);
2240 #endif
2241
2242 const TId isr_tid = TID_ISR_N | num | interrupt_bit;
2243 STK_ASSERT(IsIsrTid(isr_tid));
2244 result = isr_tid;
2245 }
2246 else
2247 {
2248 const Context &ctx = GetContext();
2249
2250 if (ctx.m_started)
2251 {
2252 result = ctx.m_handler->OnGetTid(HW_GetCallerSP());
2253 }
2254 else
2255 {
2256 result = TID_NONE;
2257 }
2258 }
2259
2260 return result;
2261}
2262
2264{
2265#if STK_TICKLESS_IDLE
2266 return GetContext().Suspend();
2267#else
2268 return 0;
2269#endif
2270}
2271
2272void PlatformRiscV::Resume(Timeout elapsed_ticks)
2273{
2274#if STK_TICKLESS_IDLE
2275 GetContext().Resume(elapsed_ticks);
2276#else
2277 STK_UNUSED(elapsed_ticks);
2278#endif
2279}
2280
2282{
2283 bool is_handled = false;
2284 const Context &ctx = GetContext();
2285
2286 if (ctx.m_overrider != nullptr)
2287 {
2288 is_handled = ctx.m_overrider->OnHardFault();
2289 }
2290
2291 if (!is_handled)
2292 {
2294 }
2295}
2296
2297void PlatformRiscV::SetEventOverrider(IEventOverrider *overrider, bool non_secure)
2298{
2299 STK_UNUSED(non_secure);
2300
2301 STK_ASSERT(!GetContext().m_started);
2302 GetContext().m_overrider = overrider;
2303}
2304
2306{
2307 return HW_GetCallerSP();
2308}
2309
2310void PlatformRiscV::SetCpuFrequency(uint8_t core_id, uint32_t frequency)
2311{
2313
2314 if (core_id == 0xFFU)
2315 {
2316 for (uint8_t i = 0U; i < STK_STATIC_ARRAY_SIZE(s_StkSystemCoreClock); ++i)
2317 {
2318 s_StkSystemCoreClock[i] = frequency;
2319 }
2320 }
2321 else
2322 {
2323 STK_ASSERT(core_id < STK_STATIC_ARRAY_SIZE(s_StkSystemCoreClock));
2324
2325 if (core_id < STK_STATIC_ARRAY_SIZE(s_StkSystemCoreClock))
2326 {
2327 s_StkSystemCoreClock[core_id] = frequency;
2328 }
2329 }
2330}
2331
2332void PlatformRiscV::SetSpecificEventHandler(ISpecificEventHandler *handler)
2333{
2334 STK_ASSERT(!GetContext().m_started);
2335 GetContext().m_specific = handler;
2336}
2337
2339{
2340 return GetContext().m_service;
2341}
2342
2344{
2345 STK_UNUSED(is_npriv);
2346 GetContext().EnterCriticalSection();
2347 return DEFAULT_SESSION;
2348}
2349
2351{
2352 STK_UNUSED(is_npriv);
2353 GetContext().ExitCriticalSection();
2354}
2355
2357{
2358 HW_SpinLockLock(m_lock);
2359}
2360
2362{
2363 HW_SpinLockUnlock(m_lock);
2364}
2365
2367{
2368 return HW_SpinLockTryLock(m_lock);
2369}
2370
2372{
2373 return HW_IsHandlerMode();
2374}
2375
2377{
2378 // always Privileged on RISC-V
2379 return true;
2380}
2381
2382stk::hw::HiResClock::HiResClock() : HiResClockState()
2383{}
2384
2386{
2387#if STK_SUBMICORSECOND_PRECISION_TIMER
2388 return HW_GetCycleCounter();
2389#else
2390 return HW_GetMtime();
2391#endif
2392}
2393
2394uint32_t stk::hw::HiResClock::GetFrequency() const
2395{
2396#if STK_SUBMICORSECOND_PRECISION_TIMER
2397 const uint32_t freq = HW_CoreClockFrequency();
2398#else
2399 const uint32_t freq = HW_MtimeClockFrequency();
2400#endif
2401 STK_ASSERT(freq != 0U);
2402 return freq;
2403}
2404
2405#if STK_TLS && !STK_INLINE_TLS
2406Word stk::hw::GetTls()
2407{
2408 return GetContext().GetTls();
2409}
2410
2411void stk::hw::SetTls(Word tp)
2412{
2413 GetContext().SetTls(tp);
2414}
2415#endif // STK_TLS && !STK_INLINE_TLS
2416
2417#endif // _STK_ARCH_RISC_V
#define STK_SYSTEM_CORE_CLOCK_FREQUENCY
System clock frequency in Hz. Default: 150 MHz.
Contains common inventory for platform implementation.
#define STK_ARCH_GET_CPU_ID()
Get CPU core id of the caller, e.g. if called while running on core 0 then returned value must be 0.
#define GetContext()
Get platform's context.
Hardware Abstraction Layer (HAL) declarations for the stk::hw namespace.
void STK_PANIC_HANDLER_DEFAULT(stk::EKernelPanicId id)
Default panic handler: disable interrupts, record the id, and spin in a tight loop - a defined,...
#define STK_MPU_KERNEL_DATA_SECTION
Definition stk_arch.h:161
#define STK_MPU_KERNEL_BSS_SECTION
Definition stk_arch.h:163
#define STK_UNUSED(X)
Explicitly marks a variable as unused to suppress compiler warnings.
Definition stk_defs.h:715
#define __stk_attr_used
Marks a symbol as used, preventing the linker from discarding it even if no references are visible (d...
Definition stk_defs.h:339
#define __stk_forceinline
Forces compiler to always inline the decorated function, regardless of optimisation level.
Definition stk_defs.h:270
#define STK_SLEEP_TRAP_STACK_SIZE
Stack size for the sleep trap in elements of Word (default: STK_STACK_SIZE_MIN).
Definition stk_defs.h:659
#define STK_CS_NESTINGS_MAX
Maximum allowable recursion depth for critical section entry (default: 16).
Definition stk_defs.h:597
#define STK_ASSERT(e)
Runtime assertion. Halts execution if the expression e evaluates to false.
Definition stk_defs.h:532
#define STK_STACK_SIZE_MIN
Minimum stack size in elements of Word, shared by all stack allocation lower-bound checks.
Definition stk_defs.h:646
#define __stk_attr_noinline
Prevents compiler from inlining the decorated function (function prefix).
Definition stk_defs.h:350
#define STK_STATIC_ASSERT_DESC_N(NAME, X, DESC)
Compile-time assertion with a name tag and a custom error description.
Definition stk_defs.h:543
#define STK_ARCH_CPU_COUNT
Number of physical CPU cores available to the scheduler (default: 1).
Definition stk_defs.h:606
#define __stk_attr_naked
Suppresses compiler-generated function prologue and epilogue (function prefix).
Definition stk_defs.h:306
#define STK_STATIC_ASSERT_DESC(X, DESC)
Compile-time assertion with a custom error description. Produces a compilation error if X is false.
Definition stk_defs.h:548
#define STK_STACK_MEMORY_FILLER
Sentinel value written to the entire stack region at initialization (stack watermark pattern).
Definition stk_defs.h:570
#define __stk_attr_noreturn
Declares that function never returns to its caller (function prefix).
Definition stk_defs.h:317
#define STK_STATIC_ARRAY_SIZE(ARRAY)
Get size of the static array.
Definition stk_defs.h:742
#define STK_STATIC_ASSERT(X)
Compile-time assertion. Produces a compilation error if X is false.
Definition stk_defs.h:560
Contains helper implementations which simplify user-side code.
Namespace of STK package.
uintptr_t Word
Native processor word type.
Definition stk_common.h:206
static constexpr TId TID_ISR_N
Bitmask sentinel for ISR-context task identifiers.
Definition stk_common.h:263
void STK_KERNEL_PANIC(stk::EKernelPanicId id)
Called when the kernel detects an unrecoverable internal fault.
Definition stk_arch.h:184
@ ACCESS_PRIVILEGED
Privileged access mode (access to hardware is fully unrestricted).
Definition stk_common.h:38
static constexpr Timeout NO_WAIT
Timeout value: return immediately if the synchronization object is not yet signaled (non-blocking pol...
Definition stk_common.h:280
static __stk_forceinline Cycles ConvertTimeUsToClockCycles(uint32_t clock_freq, Ticks time_us)
Convert time (microseconds) to core clock cycles.
EKernelPanicId
Identifies the source of a kernel panic.
Definition stk_common.h:60
@ KERNEL_PANIC_BAD_STACK_TYPE
Stack type is unknown.
Definition stk_common.h:71
@ KERNEL_PANIC_HRT_HARD_FAULT
Kernel running in KERNEL_HRT mode reported deadline failure of the task.
Definition stk_common.h:65
@ KERNEL_PANIC_NONE
Panic is absent (no fault).
Definition stk_common.h:61
@ KERNEL_PANIC_CPU_EXCEPTION
CPU reported an exception and halted execution.
Definition stk_common.h:66
@ KERNEL_PANIC_SPINLOCK_DEADLOCK
Spin-lock timeout expired: lock owner never released.
Definition stk_common.h:62
int32_t Timeout
Timeout time (ticks).
Definition stk_common.h:216
constexpr T Max(T a, T b) noexcept
Compile-time maximum of two values.
Definition stk_defs.h:758
constexpr T Min(T a, T b) noexcept
Compile-time minimum of two values.
Definition stk_defs.h:752
static constexpr bool IsIsrTid(TId id)
Test whether a task identifier represents an ISR context.
Definition stk_common.h:309
static constexpr TId TID_NONE
Reserved task/thread id representing zero/none thread id.
Definition stk_common.h:268
EStackType
Stack type.
Definition stk_common.h:96
@ STACK_SLEEP_TRAP
Stack of the Sleep trap.
Definition stk_common.h:98
@ STACK_USER_TASK
Stack of the user task.
Definition stk_common.h:97
@ STACK_EXIT_TRAP
Stack of the Exit trap.
Definition stk_common.h:99
uint64_t Cycles
Cycles value.
Definition stk_common.h:231
Word TId
Task (thread) id.
Definition stk_common.h:211
bool IsPrivilegedContext()
Check if caller context is Privileged.
static T * WordToPtr(Word value) noexcept
Cast a CPU register-width integer back to a pointer.
Definition stk_arch.h:232
static Word PtrToWord(T *const ptr) noexcept
Cast a pointer to a CPU register-width integer.
Definition stk_arch.h:215
bool IsInsideISR()
Check whether the CPU is currently executing inside a hardware interrupt service routine (ISR).
Cycles GetSysTimerCount() const override
Get system timer count value.
Word GetCallerSP() const override
Get caller's Stack Pointer (SP).
void Resume(Timeout elapsed_ticks) override
Resume scheduling after a prior Suspend() call.
void Start() override
Start scheduling.
uint32_t GetTickResolution() const override
Get resolution of the system tick timer in microseconds. Resolution means a number of microseconds be...
uint32_t GetSysTimerFrequency() const override
Get system timer frequency.
void ProcessHardFault() override
Cause a hard fault of the system.
void InitStack(EStackType stack_type, Stack *stack, IStackMemory *stack_memory, ITask *user_task) override
Initialize stack memory of the user task.
void ProcessTick() override
Process one tick.
TId GetTid() const override
Get thread Id.
void ForceContextSwitch(TId id) override
Force context switch.
void SetSpecificEventHandler(ISpecificEventHandler *handler)
void SetCpuFrequency(uint8_t core_id, uint32_t frequency) override
Notify the scheduler of a CPU core's operating frequency.
void SetEventOverrider(IEventOverrider *overrider, bool non_secure) override
Set platform event overrider.
void Stop() override
Stop scheduling.
Timeout Suspend() override
Suspend scheduling.
void Initialize(IEventHandler *event_handler, IKernelService *service, uint32_t resolution_us, Stack *exit_trap) override
Initialize scheduler's context.
Base platform context for all platform implementations.
static Word InitStackMemory(IStackMemory *const memory)
Initialize stack memory by filling it with STK_STACK_MEMORY_FILLER.
static constexpr Session DEFAULT_SESSION
Default session value passed to Enter()/Exit() when the caller does not need to force a specific hand...
Definition stk_arch.h:479
static Session Enter(const Session ses=DEFAULT_SESSION)
Enter a critical section.
uint8_t Session
Opaque session token returned by Enter() and consumed by Exit().
Definition stk_arch.h:472
static void Exit(const Session ses=DEFAULT_SESSION)
Exit a critical section.
RAII instance that enters the critical section on construction and exits it on destruction.
Definition stk_arch.h:495
bool TryLock()
Attempt to acquire SpinLock in a single non-blocking attempt.
void Lock()
Acquire SpinLock, blocking until it is available.
void Unlock()
Release SpinLock, allowing another thread or core to acquire it.
HiResClock()
Construct a clock instance.
uint32_t GetFrequency() const
Get clock frequency.
Cycles GetCycles()
Get number of clock cycles elapsed since the time origin (see the class description).
Stack descriptor.
Definition stk_common.h:473
Word SP
Offset 0: Stack Pointer (SP) register (note: must always be at offset 0).
Definition stk_common.h:474
Interface for a stack memory region.
Definition stk_common.h:494
virtual size_t GetStackSize() const =0
Get number of elements of the stack memory array.
Interface for a user task.
Definition stk_common.h:886
virtual void Run()=0
Entry point of the user task.
Interface for the kernel services exposed to the user processes during run-time when Kernel started s...
static IKernelService * GetInstance()
Get CPU-local instance of the kernel service.
static uint8_t stack_app_main[1024U] __attribute__((aligned(16)))