From 3725e91603cf322fecc234278ded7ac2be85836e Mon Sep 17 00:00:00 2001 From: Stefan Kalkowski Date: Wed, 13 Mar 2019 14:58:23 +0100 Subject: [PATCH] hw: implement power-saving kernel lock for ARM smp Thanks to former work of Martin Stein this commit finally incorporates a non-spinning kernel lock on multi-core ARM platforms. Fix #1313 --- repos/base-hw/lib/mk/core-hw.inc | 1 - repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc | 1 + .../lib/mk/spec/cortex_a15/core-hw.inc | 1 + .../base-hw/lib/mk/spec/cortex_a8/core-hw.inc | 1 + .../base-hw/lib/mk/spec/cortex_a9/core-hw.inc | 1 + repos/base-hw/lib/mk/spec/muen/core-hw.mk | 1 + repos/base-hw/lib/mk/spec/riscv/core-hw.mk | 1 + repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk | 1 + .../src/bootstrap/spec/arm/cortex_a9_mmu.cc | 20 +++++--- repos/base-hw/src/core/kernel/lock.cc | 13 +++-- repos/base-hw/src/core/kernel/lock.h | 8 +-- .../base-hw/src/core/spec/arm/kernel/lock.cc | 50 +++++++++++++++++++ repos/base-hw/src/lib/hw/spec/arm/cpu.h | 30 +++++++++++ repos/base-hw/src/lib/hw/spin_lock.h | 49 ------------------ 14 files changed, 114 insertions(+), 64 deletions(-) create mode 100644 repos/base-hw/src/core/spec/arm/kernel/lock.cc delete mode 100644 repos/base-hw/src/lib/hw/spin_lock.h diff --git a/repos/base-hw/lib/mk/core-hw.inc b/repos/base-hw/lib/mk/core-hw.inc index 0535995f1..d06c203d7 100644 --- a/repos/base-hw/lib/mk/core-hw.inc +++ b/repos/base-hw/lib/mk/core-hw.inc @@ -58,7 +58,6 @@ SRC_CC += kernel/init.cc SRC_CC += kernel/ipc_node.cc SRC_CC += kernel/irq.cc SRC_CC += kernel/kernel.cc -SRC_CC += kernel/lock.cc SRC_CC += kernel/object.cc SRC_CC += kernel/signal_receiver.cc SRC_CC += kernel/thread.cc diff --git a/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc b/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc index 1d115b886..62024258f 100644 --- a/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc @@ -12,6 +12,7 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/arm_v6 SRC_CC += spec/arm_v6/perf_counter.cc SRC_CC += kernel/vm_thread_off.cc SRC_CC += kernel/cpu_up.cc +SRC_CC += kernel/lock.cc SRC_S += spec/arm/vfpv2.s diff --git a/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc b/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc index 540436420..bf0787559 100644 --- a/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc @@ -11,6 +11,7 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/arm_gic # add C++ sources SRC_CC += spec/cortex_a15/cpu.cc SRC_CC += kernel/cpu_mp.cc +SRC_CC += spec/arm/kernel/lock.cc # include less specific configuration include $(BASE_DIR)/../base-hw/lib/mk/spec/arm_v7/core-hw.inc diff --git a/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc b/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc index c5b563688..ce5f31f91 100644 --- a/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc @@ -10,6 +10,7 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/cortex_a8 # add C++ sources SRC_CC += spec/cortex_a8/cpu.cc SRC_CC += kernel/cpu_up.cc +SRC_CC += kernel/lock.cc NR_OF_CPUS = 1 diff --git a/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc b/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc index 3d7f5831a..2b7fdd731 100644 --- a/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc @@ -12,6 +12,7 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/arm_gic SRC_CC += spec/cortex_a9/board.cc SRC_CC += spec/cortex_a9/timer.cc SRC_CC += spec/arm_gic/pic.cc +SRC_CC += spec/arm/kernel/lock.cc SRC_CC += kernel/vm_thread_off.cc SRC_CC += kernel/cpu_mp.cc SRC_CC += kernel/kernel.cc diff --git a/repos/base-hw/lib/mk/spec/muen/core-hw.mk b/repos/base-hw/lib/mk/spec/muen/core-hw.mk index 19595ba4b..dfb588358 100644 --- a/repos/base-hw/lib/mk/spec/muen/core-hw.mk +++ b/repos/base-hw/lib/mk/spec/muen/core-hw.mk @@ -20,6 +20,7 @@ SRC_S += spec/x86_64/exception_vector.s # add C++ sources SRC_CC += kernel/cpu_up.cc SRC_CC += kernel/vm_thread_on.cc +SRC_CC += kernel/lock.cc SRC_CC += spec/x86/io_port_session_component.cc SRC_CC += spec/x86/io_port_session_support.cc SRC_CC += spec/x86_64/bios_data_area.cc diff --git a/repos/base-hw/lib/mk/spec/riscv/core-hw.mk b/repos/base-hw/lib/mk/spec/riscv/core-hw.mk index 97d7cc3ba..2bae8053c 100644 --- a/repos/base-hw/lib/mk/spec/riscv/core-hw.mk +++ b/repos/base-hw/lib/mk/spec/riscv/core-hw.mk @@ -12,6 +12,7 @@ CC_OPT += -fno-delete-null-pointer-checks SRC_CC += platform_services.cc SRC_CC += kernel/vm_thread_off.cc SRC_CC += kernel/cpu_up.cc +SRC_CC += kernel/lock.cc SRC_CC += spec/riscv/cpu.cc SRC_CC += spec/riscv/kernel/thread.cc SRC_CC += spec/riscv/kernel/cpu.cc diff --git a/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk b/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk index 51df20664..25c0174fc 100644 --- a/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk +++ b/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk @@ -15,6 +15,7 @@ SRC_S += spec/x86_64/exception_vector.s # add C++ sources SRC_CC += kernel/cpu_mp.cc SRC_CC += kernel/vm_thread_off.cc +SRC_CC += kernel/lock.cc SRC_CC += spec/x86_64/pic.cc SRC_CC += spec/x86_64/timer.cc SRC_CC += spec/x86_64/kernel/thread_exception.cc diff --git a/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc b/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc index 59451caa7..47e9c9519 100644 --- a/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc +++ b/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc @@ -12,8 +12,10 @@ */ #include +#include +#include + #include -#include #include #include @@ -29,20 +31,24 @@ class Cpu_counter { private: - Hw::Spin_lock _lock { }; - volatile int _value = 0; + enum State { UNLOCKED, LOCKED }; + + State volatile _locked { UNLOCKED }; + unsigned volatile _counter { 0 }; public: void inc() { - Hw::Spin_lock::Guard guard(_lock); + while (!Genode::cmpxchg((volatile int*)&_locked, UNLOCKED, LOCKED)) + ; + _counter++; Genode::memory_barrier(); - _value++; + _locked = UNLOCKED; } - void wait_for(int const v) { - while (_value < v) ; } + void wait_for(unsigned const v) { + while (_counter < v) ; } }; diff --git a/repos/base-hw/src/core/kernel/lock.cc b/repos/base-hw/src/core/kernel/lock.cc index 62153a4af..00a85a658 100644 --- a/repos/base-hw/src/core/kernel/lock.cc +++ b/repos/base-hw/src/core/kernel/lock.cc @@ -11,6 +11,10 @@ * under the terms of the GNU Affero General Public License version 3. */ +#include +#include +#include + #include #include #include @@ -29,9 +33,10 @@ void Kernel::Lock::lock() /* at least print an error message */ Genode::raw("Cpu ", _current_cpu, " error: re-entered lock. Kernel exception?!"); - for (;;) ; } - _lock.lock(); + + while (!Genode::cmpxchg((volatile int*)&_locked, UNLOCKED, LOCKED)) { ; } + _current_cpu = Cpu::executing_id(); } @@ -39,5 +44,7 @@ void Kernel::Lock::lock() void Kernel::Lock::unlock() { _current_cpu = INVALID; - _lock.unlock(); + + Genode::memory_barrier(); + _locked = UNLOCKED; } diff --git a/repos/base-hw/src/core/kernel/lock.h b/repos/base-hw/src/core/kernel/lock.h index f24a1b828..2874960f7 100644 --- a/repos/base-hw/src/core/kernel/lock.h +++ b/repos/base-hw/src/core/kernel/lock.h @@ -15,8 +15,6 @@ #ifndef _CORE__SPEC__SMP__KERNEL__LOCK_H_ #define _CORE__SPEC__SMP__KERNEL__LOCK_H_ -#include - namespace Kernel { class Lock; @@ -31,8 +29,10 @@ class Kernel::Lock enum { INVALID = ~0U }; - Hw::Spin_lock _lock { }; - volatile unsigned _current_cpu { INVALID }; + enum State { UNLOCKED, LOCKED }; + + State volatile _locked { UNLOCKED }; + unsigned volatile _current_cpu { INVALID }; public: diff --git a/repos/base-hw/src/core/spec/arm/kernel/lock.cc b/repos/base-hw/src/core/spec/arm/kernel/lock.cc new file mode 100644 index 000000000..3b9f7635e --- /dev/null +++ b/repos/base-hw/src/core/spec/arm/kernel/lock.cc @@ -0,0 +1,50 @@ +/* + * \brief Kernel lock for multi-processor systems + * \author Stefan Kalkowski + * \date 2018-11-20 + */ + +/* + * Copyright (C) 2019 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include +#include +#include + +#include +#include +#include + +Kernel::Lock & Kernel::data_lock() +{ + static Kernel::Lock lock; + return lock; +} + + +void Kernel::Lock::lock() +{ + /* check for the lock holder being the same cpu */ + if (_current_cpu == Cpu::executing_id()) { + /* at least print an error message */ + Genode::raw("Cpu ", _current_cpu, + " error: re-entered lock. Kernel exception?!"); + } + + Cpu::wait_for_xchg(&_locked, LOCKED, UNLOCKED); + _current_cpu = Cpu::executing_id(); +} + + +void Kernel::Lock::unlock() +{ + _current_cpu = INVALID; + + Genode::memory_barrier(); + _locked = UNLOCKED; + Cpu::wakeup_waiting_cpus(); +} diff --git a/repos/base-hw/src/lib/hw/spec/arm/cpu.h b/repos/base-hw/src/lib/hw/spec/arm/cpu.h index 549be515f..77b447499 100644 --- a/repos/base-hw/src/lib/hw/spec/arm/cpu.h +++ b/repos/base-hw/src/lib/hw/spec/arm/cpu.h @@ -265,6 +265,36 @@ struct Hw::Arm_cpu asm volatile("dsb\n" "isb\n"); } + + static inline void wait_for_xchg(volatile void * addr, + unsigned long new_value, + unsigned long expected_value) + { + asm volatile( + /* check if load value of 'addr' is as expected */ + "1: ldrex r7, [%0] \n" + "cmp r7, %2 \n" + + /* if not, wait for other CPU to send us an event */ + "wfene \n" + + /* if yes, attempt to write 'new_value' to 'addr' */ + "strexeq r7, %1, [%0] \n" + + /* if write failed, restart */ + "cmpeq r7, #0 \n" + "bne 1b \n" + "dmb \n" + :: "r"(addr), "r"(new_value), "r"(expected_value) : "cc", "r7"); + } + + static inline void wakeup_waiting_cpus() + { + asm volatile( + "dsb \n" + "sev \n" + ); + } }; #endif /* _SRC__LIB__HW__SPEC__ARM__CPU_H_ */ diff --git a/repos/base-hw/src/lib/hw/spin_lock.h b/repos/base-hw/src/lib/hw/spin_lock.h deleted file mode 100644 index eeff5a87c..000000000 --- a/repos/base-hw/src/lib/hw/spin_lock.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * \brief Spin lock used to synchronize different CPU cores - * \author Martin Stein - * \author Stefan Kalkowski - * \date 2012-11-30 - */ - -/* - * Copyright (C) 2012-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -#ifndef _SRC__LIB__HW__SPIN_LOCK_H_ -#define _SRC__LIB__HW__SPIN_LOCK_H_ - -#include -#include -#include - -namespace Hw { class Spin_lock; } - -class Hw::Spin_lock -{ - private: - - enum State { UNLOCKED, LOCKED }; - - State volatile _locked = UNLOCKED; - - public: - - void lock() - { - while (!Genode::cmpxchg((volatile int*)&_locked, UNLOCKED, LOCKED)) - ; - } - - void unlock() - { - Genode::memory_barrier(); - _locked = UNLOCKED; - } - - using Guard = Genode::Lock_guard; -}; - -#endif /* _SRC__LIB__HW__SPIN_LOCK_H_ */