diff --git a/repos/base-hw/lib/mk/core-hw.inc b/repos/base-hw/lib/mk/core-hw.inc index 21b8ac4c7..0535995f1 100644 --- a/repos/base-hw/lib/mk/core-hw.inc +++ b/repos/base-hw/lib/mk/core-hw.inc @@ -51,16 +51,18 @@ SRC_CC += env.cc SRC_CC += region_map_support.cc SRC_CC += pager.cc SRC_CC += _main.cc +SRC_CC += kernel/cpu.cc SRC_CC += kernel/cpu_scheduler.cc SRC_CC += kernel/double_list.cc SRC_CC += kernel/init.cc -SRC_CC += kernel/thread.cc -SRC_CC += kernel/signal_receiver.cc SRC_CC += kernel/ipc_node.cc SRC_CC += kernel/irq.cc -SRC_CC += kernel/cpu.cc -SRC_CC += kernel/timer.cc +SRC_CC += kernel/kernel.cc +SRC_CC += kernel/lock.cc SRC_CC += kernel/object.cc +SRC_CC += kernel/signal_receiver.cc +SRC_CC += kernel/thread.cc +SRC_CC += kernel/timer.cc SRC_CC += init_main_thread.cc SRC_CC += capability.cc SRC_CC += stack_area_addr.cc diff --git a/repos/base-hw/lib/mk/spec/arm/core-hw.inc b/repos/base-hw/lib/mk/spec/arm/core-hw.inc index 375925a20..c4e071a24 100644 --- a/repos/base-hw/lib/mk/spec/arm/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/arm/core-hw.inc @@ -9,6 +9,8 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/arm # add C++ sources SRC_CC += spec/32bit/memory_map.cc +SRC_CC += spec/arm/kernel/cpu.cc +SRC_CC += spec/arm/kernel/pd.cc SRC_CC += spec/arm/cpu.cc SRC_CC += spec/arm/kernel/thread.cc SRC_CC += spec/arm/platform_support.cc diff --git a/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc b/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc index 23a82ad64..8593236ee 100644 --- a/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/arm_v6/core-hw.inc @@ -11,10 +11,8 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/arm_v6 # add C++ sources SRC_CC += spec/arm_v6/cpu.cc SRC_CC += spec/arm_v6/perf_counter.cc -SRC_CC += spec/arm/kernel/cpu.cc -SRC_CC += spec/arm/kernel/thread_update_pd.cc SRC_CC += kernel/vm_thread_off.cc -SRC_CC += kernel/kernel.cc +SRC_CC += kernel/cpu_up.cc SRC_S += spec/arm/vfpv2.s diff --git a/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc b/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc index 29fffe701..540436420 100644 --- a/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/cortex_a15/core-hw.inc @@ -10,10 +10,7 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/arm_gic # add C++ sources SRC_CC += spec/cortex_a15/cpu.cc -SRC_CC += spec/cortex_a15/kernel/cpu.cc -SRC_CC += spec/arm/smp/kernel/thread_update_pd.cc -SRC_CC += spec/arm/smp/kernel/cpu.cc +SRC_CC += kernel/cpu_mp.cc # include less specific configuration -include $(BASE_DIR)/../base-hw/lib/mk/spec/smp/core-hw.inc include $(BASE_DIR)/../base-hw/lib/mk/spec/arm_v7/core-hw.inc diff --git a/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc b/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc index 5bccf8caa..c5b563688 100644 --- a/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/cortex_a8/core-hw.inc @@ -9,9 +9,7 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/cortex_a8 # add C++ sources SRC_CC += spec/cortex_a8/cpu.cc -SRC_CC += spec/arm/kernel/cpu.cc -SRC_CC += spec/arm/kernel/thread_update_pd.cc -SRC_CC += kernel/kernel.cc +SRC_CC += kernel/cpu_up.cc NR_OF_CPUS = 1 diff --git a/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc b/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc index 106f92d94..3d7f5831a 100644 --- a/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc +++ b/repos/base-hw/lib/mk/spec/cortex_a9/core-hw.inc @@ -9,14 +9,12 @@ INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/cortex_a9 INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/arm_gic # add C++ sources -SRC_CC += spec/cortex_a9/kernel/cpu.cc SRC_CC += spec/cortex_a9/board.cc SRC_CC += spec/cortex_a9/timer.cc -SRC_CC += spec/arm/smp/kernel/thread_update_pd.cc -SRC_CC += spec/arm/smp/kernel/cpu.cc SRC_CC += spec/arm_gic/pic.cc SRC_CC += kernel/vm_thread_off.cc +SRC_CC += kernel/cpu_mp.cc +SRC_CC += kernel/kernel.cc # include less specific configuration -include $(BASE_DIR)/../base-hw/lib/mk/spec/smp/core-hw.inc include $(BASE_DIR)/../base-hw/lib/mk/spec/arm_v7/core-hw.inc diff --git a/repos/base-hw/lib/mk/spec/muen/core-hw.mk b/repos/base-hw/lib/mk/spec/muen/core-hw.mk index f354fed3a..19595ba4b 100644 --- a/repos/base-hw/lib/mk/spec/muen/core-hw.mk +++ b/repos/base-hw/lib/mk/spec/muen/core-hw.mk @@ -18,23 +18,23 @@ SRC_S += spec/x86_64/crt0.s SRC_S += spec/x86_64/exception_vector.s # add C++ sources -SRC_CC += spec/x86_64/muen/kernel/thread_exception.cc -SRC_CC += spec/x86_64/muen/platform_support.cc -SRC_CC += spec/x86_64/muen/kernel/vm.cc -SRC_CC += spec/x86_64/muen/platform_services.cc -SRC_CC += spec/x86_64/muen/sinfo_instance.cc -SRC_CC += spec/x86_64/muen/timer.cc +SRC_CC += kernel/cpu_up.cc SRC_CC += kernel/vm_thread_on.cc - -SRC_CC += kernel/kernel.cc SRC_CC += spec/x86/io_port_session_component.cc SRC_CC += spec/x86/io_port_session_support.cc SRC_CC += spec/x86_64/bios_data_area.cc SRC_CC += spec/x86_64/cpu.cc SRC_CC += spec/x86_64/fpu.cc SRC_CC += spec/x86_64/kernel/cpu.cc +SRC_CC += spec/x86_64/kernel/pd.cc SRC_CC += spec/x86_64/kernel/thread.cc SRC_CC += spec/x86_64/kernel/thread.cc +SRC_CC += spec/x86_64/muen/kernel/thread_exception.cc +SRC_CC += spec/x86_64/muen/kernel/vm.cc +SRC_CC += spec/x86_64/muen/platform_services.cc +SRC_CC += spec/x86_64/muen/platform_support.cc +SRC_CC += spec/x86_64/muen/sinfo_instance.cc +SRC_CC += spec/x86_64/muen/timer.cc SRC_CC += spec/x86_64/platform_support_common.cc SRC_CC += spec/64bit/memory_map.cc diff --git a/repos/base-hw/lib/mk/spec/riscv/core-hw.mk b/repos/base-hw/lib/mk/spec/riscv/core-hw.mk index 0f489769c..fe940bcd7 100644 --- a/repos/base-hw/lib/mk/spec/riscv/core-hw.mk +++ b/repos/base-hw/lib/mk/spec/riscv/core-hw.mk @@ -4,10 +4,12 @@ CC_OPT += -fno-delete-null-pointer-checks # add C++ sources SRC_CC += platform_services.cc -SRC_CC += kernel/vm_thread_off.cc kernel/kernel.cc +SRC_CC += kernel/vm_thread_off.cc +SRC_CC += kernel/cpu_up.cc SRC_CC += spec/riscv/cpu.cc SRC_CC += spec/riscv/kernel/thread.cc SRC_CC += spec/riscv/kernel/cpu.cc +SRC_CC += spec/riscv/kernel/pd.cc SRC_CC += spec/riscv/platform_support.cc SRC_CC += spec/riscv/timer.cc SRC_CC += spec/64bit/memory_map.cc diff --git a/repos/base-hw/lib/mk/spec/smp/core-hw.inc b/repos/base-hw/lib/mk/spec/smp/core-hw.inc deleted file mode 100644 index cf0230590..000000000 --- a/repos/base-hw/lib/mk/spec/smp/core-hw.inc +++ /dev/null @@ -1,12 +0,0 @@ -# -# \brief Build config for Genodes core process -# \author Stefan Kalkowski -# \date 2016-01-04 -# - -# add include paths -INC_DIR += $(BASE_DIR)/../base-hw/src/core/spec/smp - -# add C++ sources -SRC_CC += spec/smp/kernel/kernel.cc -SRC_CC += spec/smp/kernel/cpu.cc diff --git a/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk b/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk index d45685b2c..51df20664 100644 --- a/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk +++ b/repos/base-hw/lib/mk/spec/x86_pc/core-hw.mk @@ -13,6 +13,7 @@ SRC_S += spec/x86_64/crt0.s SRC_S += spec/x86_64/exception_vector.s # add C++ sources +SRC_CC += kernel/cpu_mp.cc SRC_CC += kernel/vm_thread_off.cc SRC_CC += spec/x86_64/pic.cc SRC_CC += spec/x86_64/timer.cc @@ -26,10 +27,10 @@ SRC_CC += spec/x86_64/bios_data_area.cc SRC_CC += spec/x86_64/cpu.cc SRC_CC += spec/x86_64/fpu.cc SRC_CC += spec/x86_64/kernel/cpu.cc +SRC_CC += spec/x86_64/kernel/pd.cc SRC_CC += spec/x86_64/kernel/thread.cc SRC_CC += spec/x86_64/kernel/thread.cc SRC_CC += spec/x86_64/platform_support_common.cc -SRC_CC += spec/x86_64/smp/cpu.cc SRC_CC += spec/64bit/memory_map.cc @@ -39,4 +40,3 @@ NR_OF_CPUS = 32 # include less specific configuration include $(BASE_DIR)/../base-hw/lib/mk/core-hw.inc -include $(BASE_DIR)/../base-hw/lib/mk/spec/smp/core-hw.inc diff --git a/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc b/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc index 5ba69c79d..59451caa7 100644 --- a/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc +++ b/repos/base-hw/src/bootstrap/spec/arm/cortex_a9_mmu.cc @@ -164,5 +164,6 @@ unsigned Bootstrap::Platform::enable_mmu() /* wait for other cores' coherency activation */ smp_coherency_enabled.wait_for(NR_OF_CPUS); + Cpu::synchronization_barrier(); return Cpu::Mpidr::Aff_0::get(Cpu::Mpidr::read()); } diff --git a/repos/base-hw/src/core/kernel/cpu.cc b/repos/base-hw/src/core/kernel/cpu.cc index d7267b38a..701cbdd11 100644 --- a/repos/base-hw/src/core/kernel/cpu.cc +++ b/repos/base-hw/src/core/kernel/cpu.cc @@ -20,6 +20,7 @@ #include #include #include +#include /* base-internal includes */ #include @@ -194,36 +195,34 @@ addr_t Cpu::stack_start() { return (addr_t)&kernel_stack + KERNEL_STACK_SIZE * (_id+1); } -Cpu::Cpu(unsigned const id) +Cpu::Cpu(unsigned const id, Pic & pic, + Inter_processor_work_list & global_work_list) : - _id(id), _timer(_id), + _id(id), _pic(pic), _timer(_id), _scheduler(&_idle, _quota(), _fill()), _idle(this), - _ipi_irq(*this), _timer_irq(_timer.interrupt_id(), *this) -{ } + _ipi_irq(*this), _timer_irq(_timer.interrupt_id(), *this), + _global_work_list(global_work_list) +{ _arch_init(); } /************** ** Cpu_pool ** **************/ -Cpu * Cpu_pool::cpu(unsigned const id) const +bool Cpu_pool::initialize(Pic & pic) { - assert(id < NR_OF_CPUS); - char * const p = const_cast(_cpus[id]); - return reinterpret_cast(p); + unsigned id = Cpu::executing_id(); + _cpus[id].construct(id, pic, _global_work_list); + return --_initialized == 0; +} + + +Cpu & Cpu_pool::cpu(unsigned const id) +{ + assert(id < _count && _cpus[id].constructed()); + return *_cpus[id]; } Cpu_pool::Cpu_pool() -{ - for (unsigned id = 0; id < NR_OF_CPUS; id++) { - new (_cpus[id]) Cpu(id); } -} - - -/*********************** - ** Cpu_domain_update ** - ***********************/ - -Cpu_domain_update::Cpu_domain_update() { - for (unsigned i = 0; i < NR_OF_CPUS; i++) { _pending[i] = false; } } +: _count(reinterpret_cast(Hw::Mm::boot_info().base)->cpus) { } diff --git a/repos/base-hw/src/core/kernel/cpu.h b/repos/base-hw/src/core/kernel/cpu.h index 42187acbe..274c40fbb 100644 --- a/repos/base-hw/src/core/kernel/cpu.h +++ b/repos/base-hw/src/core/kernel/cpu.h @@ -15,9 +15,12 @@ #ifndef _CORE__KERNEL__CPU_H_ #define _CORE__KERNEL__CPU_H_ +#include + /* core includes */ #include #include +#include #include namespace Kernel @@ -67,7 +70,6 @@ namespace Kernel */ #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" - class Kernel::Cpu : public Genode::Cpu, private Irq::Pool, private Timeout { private: @@ -79,7 +81,15 @@ class Kernel::Cpu : public Genode::Cpu, private Irq::Pool, private Timeout */ struct Ipi : Irq { - bool pending = false; + Cpu & cpu; + bool pending { false }; + + /** + * Constructor + * + * \param cpu cpu this IPI belongs to + */ + Ipi(Cpu & cpu); /********************* @@ -87,22 +97,9 @@ class Kernel::Cpu : public Genode::Cpu, private Irq::Pool, private Timeout *********************/ void occurred(); - - /** - * Constructor - * - * \param p interrupt pool this irq shall reside in - */ - Ipi(Irq::Pool &p); - - /** - * Trigger the ipi - * - * \param cpu_id id of the cpu this ipi object is related to - */ - void trigger(unsigned const cpu_id); }; + friend void Ipi::occurred(void); struct Idle_thread : Kernel::Thread { @@ -114,12 +111,17 @@ class Kernel::Cpu : public Genode::Cpu, private Irq::Pool, private Timeout unsigned const _id; + Pic &_pic; Timer _timer; Cpu_scheduler _scheduler; Idle_thread _idle; Ipi _ipi_irq; Irq _timer_irq; /* timer IRQ implemented as empty event */ + Inter_processor_work_list &_global_work_list; + Inter_processor_work_list _local_work_list {}; + + void _arch_init(); unsigned _quota() const { return _timer.us_to_ticks(cpu_quota_us); } unsigned _fill() const { return _timer.us_to_ticks(cpu_fill_us); } @@ -130,21 +132,13 @@ class Kernel::Cpu : public Genode::Cpu, private Irq::Pool, private Timeout /** * Construct object for CPU 'id' */ - Cpu(unsigned const id); - - /** - * Initialize primary cpu object - * - * \param pic interrupt controller object - * \param core_pd core's pd object - * \param board object encapsulating board specifics - */ - void init(Pic &pic/*, Kernel::Pd &core_pd, Genode::Board & board*/); + Cpu(unsigned const id, Pic & pic, + Inter_processor_work_list & global_work_list); /** * Raise the IPI of the CPU */ - void trigger_ip_interrupt() { _ipi_irq.trigger(_id); } + void trigger_ip_interrupt(); /** * Deliver interrupt to the CPU @@ -188,6 +182,9 @@ class Kernel::Cpu : public Genode::Cpu, private Irq::Pool, private Timeout unsigned timer_interrupt_id() const { return _timer.interrupt_id(); } Irq::Pool &irq_pool() { return *this; } + + Inter_processor_work_list & work_list() { + return _local_work_list; } }; @@ -201,39 +198,40 @@ class Kernel::Cpu_pool { private: - /* - * Align to machine word size, otherwise load/stores might fail on some - * platforms. - */ - char _cpus[NR_OF_CPUS][sizeof(Cpu)] - __attribute__((aligned(sizeof(addr_t)))); + Inter_processor_work_list _global_work_list {}; + unsigned _count; + unsigned _initialized { _count }; + Genode::Constructible _cpus[NR_OF_CPUS]; public: Cpu_pool(); + bool initialize(Pic & pic); + /** * Return object of CPU 'id' */ - Cpu * cpu(unsigned const id) const; + Cpu & cpu(unsigned const id); /** * Return object of primary CPU */ - Cpu * primary_cpu() const { return cpu(Cpu::primary_id()); } + Cpu & primary_cpu() { return cpu(Cpu::primary_id()); } /** * Return object of current CPU */ - Cpu * executing_cpu() const { return cpu(Cpu::executing_id()); } + Cpu & executing_cpu() { return cpu(Cpu::executing_id()); } template - void for_each_cpu(FUNC const &func) const + void for_each_cpu(FUNC const &func) { - for (unsigned i = 0; i < sizeof(_cpus)/sizeof(_cpus[i]); i++) { - func(*cpu(i)); - } + for (unsigned i = 0; i < _count; i++) func(cpu(i)); } + + Inter_processor_work_list & work_list() { + return _global_work_list; } }; #endif /* _CORE__KERNEL__CPU_H_ */ diff --git a/repos/base-hw/src/core/kernel/cpu_context.h b/repos/base-hw/src/core/kernel/cpu_context.h index b34aa127e..792a37569 100644 --- a/repos/base-hw/src/core/kernel/cpu_context.h +++ b/repos/base-hw/src/core/kernel/cpu_context.h @@ -27,50 +27,8 @@ namespace Kernel * Context of a job (thread, VM, idle) that shall be executed by a CPU */ class Cpu_job; - - /** - * Ability to do a domain update on all CPUs - */ - class Cpu_domain_update; } -class Kernel::Cpu_domain_update : private Double_list_item -{ - friend class Cpu_domain_update_list; - friend class Kernel::Double_list_typed; - - private: - - bool _pending[NR_OF_CPUS]; - unsigned _domain_id = 0; - - /** - * Domain-update back-end - */ - void _domain_update(); - - /** - * Perform the domain update on the executing CPU - */ - void _do(); - - protected: - - Cpu_domain_update(); - - virtual ~Cpu_domain_update() { }; - - /** - * Do an update of domain 'id' on all CPUs and return if this blocks - */ - bool _do_global(unsigned const id); - - /** - * Notice that the update isn't pending on any CPU anymore - */ - virtual void _cpu_domain_update_unblocks() = 0; -}; - class Kernel::Cpu_job : private Cpu_share { private: diff --git a/repos/base-hw/src/core/kernel/cpu_mp.cc b/repos/base-hw/src/core/kernel/cpu_mp.cc new file mode 100644 index 000000000..ced77609d --- /dev/null +++ b/repos/base-hw/src/core/kernel/cpu_mp.cc @@ -0,0 +1,49 @@ +/* + * \brief Kernel cpu object implementations for multiprocessor systems + * \author Stefan Kalkowski + * \date 2018-11-18 + */ + +/* + * Copyright (C) 2018 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include + +using namespace Kernel; + +void Cpu::Ipi::occurred() +{ + /* lambda to iterate over a work-list and execute all work items */ + auto iterate = [] (Genode::List> & li) { + Genode::List_element const *e = li.first(); + Genode::List_element const *next = nullptr; + for ( ; e; e = next) { + next = e->next(); + e->object()->execute(); + } + }; + + /* iterate through the local and global work-list */ + iterate(cpu._local_work_list); + iterate(cpu._global_work_list); + + /* mark the IPI as being received */ + pending = false; +} + + +void Cpu::trigger_ip_interrupt() +{ + /* check whether there is still an IPI send */ + if (_ipi_irq.pending) return; + + pic()->send_ipi(_id); + _ipi_irq.pending = true; +} + + +Cpu::Ipi::Ipi(Cpu & cpu) : Irq(Pic::IPI, cpu), cpu(cpu) { } diff --git a/repos/base-hw/src/core/kernel/cpu_up.cc b/repos/base-hw/src/core/kernel/cpu_up.cc new file mode 100644 index 000000000..8c1d06d11 --- /dev/null +++ b/repos/base-hw/src/core/kernel/cpu_up.cc @@ -0,0 +1,22 @@ +/* + * \brief Kernel cpu object implementations for uniprocessors + * \author Stefan Kalkowski + * \date 2018-11-08 + */ + +/* + * Copyright (C) 2018 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include + +void Kernel::Cpu::Ipi::occurred() { } + + +void Kernel::Cpu::trigger_ip_interrupt() { } + + +Kernel::Cpu::Ipi::Ipi(Kernel::Cpu & cpu) : Irq(~0U, cpu), cpu(cpu) { } diff --git a/repos/base-hw/src/core/kernel/init.cc b/repos/base-hw/src/core/kernel/init.cc index 20195fbc7..85a9f3a86 100644 --- a/repos/base-hw/src/core/kernel/init.cc +++ b/repos/base-hw/src/core/kernel/init.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -42,22 +43,32 @@ extern "C" void kernel_init(); */ extern "C" void kernel_init() { - static volatile bool initialized = false; - if (Cpu::executing_id()) while (!initialized) ; - else { + static volatile bool pool_ready = false; + static volatile bool kernel_ready = false; + + { + Lock::Guard guard(data_lock()); + + /* initialize current cpu */ + pool_ready = cpu_pool()->initialize(*pic()); + }; + + /* wait until all cpus have initialized their corresponding cpu object */ + while (!pool_ready) { ; } + + /* the boot-cpu initializes the rest of the kernel */ + if (Cpu::executing_id() == Cpu::primary_id()) { + Lock::Guard guard(data_lock()); + Genode::log(""); Genode::log("kernel initialized"); + + Core_thread::singleton(); + kernel_ready = true; + } else { + /* secondary cpus spin until the kernel is initialized */ + while (!kernel_ready) {;} } - /* initialize cpu pool */ - cpu_pool(); - - /* initialize current cpu */ - cpu_pool()->cpu(Cpu::executing_id())->init(*pic()); - - Core_thread::singleton(); - - if (!Cpu::executing_id()) initialized = true; - kernel(); } diff --git a/repos/base-hw/src/core/kernel/inter_processor_work.h b/repos/base-hw/src/core/kernel/inter_processor_work.h new file mode 100644 index 000000000..b6a4f2fdc --- /dev/null +++ b/repos/base-hw/src/core/kernel/inter_processor_work.h @@ -0,0 +1,42 @@ +/* + * \brief Kernel interface for inter-processor communication + * \author Stefan Kalkowski + * \date 2018-11-15 + */ + +/* + * Copyright (C) 2012-2017 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#ifndef _CORE__KERNEL__SMP_H_ +#define _CORE__KERNEL__SMP_H_ + +#include + +namespace Kernel { + + /** + * Work that has to be propagated to a different cpu resp. core + */ + class Inter_processor_work; + + using Inter_processor_work_list = + Genode::List >; +} + + +class Kernel::Inter_processor_work : Genode::Interface +{ + public: + + virtual void execute() = 0; + + protected: + + Genode::List_element _le { this }; +}; + +#endif /* _CORE__KERNEL__SMP_H_ */ diff --git a/repos/base-hw/src/core/kernel/kernel.cc b/repos/base-hw/src/core/kernel/kernel.cc index c94b233cc..4f2d8b1cb 100644 --- a/repos/base-hw/src/core/kernel/kernel.cc +++ b/repos/base-hw/src/core/kernel/kernel.cc @@ -1,5 +1,5 @@ /* - * \brief Kernel entrypoint for non-SMP systems + * \brief Kernel entrypoint * \author Martin Stein * \author Stefan Kalkowski * \date 2011-10-20 @@ -14,21 +14,22 @@ /* core includes */ #include +#include +#include extern "C" void kernel() { using namespace Kernel; - Cpu * const cpu = cpu_pool()->cpu(Cpu::executing_id()); - cpu->schedule().proceed(*cpu); + Cpu & cpu = cpu_pool()->cpu(Cpu::executing_id()); + Cpu_job * new_job; + + { + Lock::Guard guard(data_lock()); + + new_job = &cpu.schedule(); + } + + new_job->proceed(cpu); } - - -void Kernel::Cpu::Ipi::occurred() { } - - -void Kernel::Cpu::Ipi::trigger(unsigned) { } - - -Kernel::Cpu::Ipi::Ipi(Kernel::Irq::Pool &p) : Kernel::Irq(Kernel::Pic::IPI, p) { } diff --git a/repos/base-hw/src/core/kernel/lock.cc b/repos/base-hw/src/core/kernel/lock.cc new file mode 100644 index 000000000..62153a4af --- /dev/null +++ b/repos/base-hw/src/core/kernel/lock.cc @@ -0,0 +1,43 @@ +/* + * \brief Kernel lock for multi-processor systems + * \author Stefan Kalkowski + * \date 2018-11-20 + */ + +/* + * Copyright (C) 2019 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include +#include +#include + +Kernel::Lock & Kernel::data_lock() +{ + static Kernel::Lock lock; + return lock; +} + + +void Kernel::Lock::lock() +{ + /* check for the lock holder being the same cpu */ + if (_current_cpu == Cpu::executing_id()) { + /* at least print an error message */ + Genode::raw("Cpu ", _current_cpu, + " error: re-entered lock. Kernel exception?!"); + for (;;) ; + } + _lock.lock(); + _current_cpu = Cpu::executing_id(); +} + + +void Kernel::Lock::unlock() +{ + _current_cpu = INVALID; + _lock.unlock(); +} diff --git a/repos/base-hw/src/core/spec/smp/kernel/lock.h b/repos/base-hw/src/core/kernel/lock.h similarity index 67% rename from repos/base-hw/src/core/spec/smp/kernel/lock.h rename to repos/base-hw/src/core/kernel/lock.h index 9787f5d02..f24a1b828 100644 --- a/repos/base-hw/src/core/spec/smp/kernel/lock.h +++ b/repos/base-hw/src/core/kernel/lock.h @@ -19,9 +19,27 @@ namespace Kernel { - using Lock = Hw::Spin_lock; + class Lock; Lock & data_lock(); } + +class Kernel::Lock +{ + private: + + enum { INVALID = ~0U }; + + Hw::Spin_lock _lock { }; + volatile unsigned _current_cpu { INVALID }; + + public: + + void lock(); + void unlock(); + + using Guard = Genode::Lock_guard; +}; + #endif /* _CORE__SPEC__SMP__KERNEL__LOCK_H_ */ diff --git a/repos/base-hw/src/core/kernel/pd.h b/repos/base-hw/src/core/kernel/pd.h index 1214b1f37..af464b106 100644 --- a/repos/base-hw/src/core/kernel/pd.h +++ b/repos/base-hw/src/core/kernel/pd.h @@ -28,6 +28,8 @@ namespace Genode { namespace Kernel { + class Cpu; + /** * Kernel backend of protection domains */ @@ -81,7 +83,6 @@ class Kernel::Pd : public Kernel::Object oir->~Object_identity_reference(); } - static capid_t syscall_create(void * const dst, Hw::Page_table * tt, Genode::Platform_pd * const pd) @@ -93,6 +94,12 @@ class Kernel::Pd : public Kernel::Object static void syscall_destroy(Pd * const pd) { call(call_id_delete_pd(), (Call_arg)pd); } + /** + * Check whether the given 'cpu' needs to do some maintainance + * work, after this pd has had changes in its page-tables + */ + bool update(Cpu & cpu); + /*************** ** Accessors ** diff --git a/repos/base-hw/src/core/kernel/thread.cc b/repos/base-hw/src/core/kernel/thread.cc index 20ef5781c..6a8ba8fb5 100644 --- a/repos/base-hw/src/core/kernel/thread.cc +++ b/repos/base-hw/src/core/kernel/thread.cc @@ -38,6 +38,30 @@ extern "C" void _core_start(void); using namespace Kernel; +Thread::Pd_update::Pd_update(Thread & caller, Pd & pd, unsigned cnt) +: caller(caller), pd(pd), cnt(cnt) +{ + cpu_pool()->work_list().insert(&_le); + caller._become_inactive(AWAITS_RESTART); +} + + +Thread::Destroy::Destroy(Thread & caller, Thread & to_delete) +: caller(caller), thread_to_destroy(to_delete) +{ + thread_to_destroy._cpu->work_list().insert(&_le); + caller._become_inactive(AWAITS_RESTART); +} + + +void Thread::Destroy::execute() +{ + thread_to_destroy.~Thread(); + cpu_pool()->executing_cpu().work_list().remove(&_le); + caller._restart(); +} + + void Thread_fault::print(Genode::Output &out) const { Genode::print(out, "ip=", Genode::Hex(ip)); @@ -200,18 +224,13 @@ void Thread::_call_thread_quota() void Thread::_call_start_thread() { /* lookup CPU */ - Cpu * const cpu = cpu_pool()->cpu(user_arg_2()); - if (!cpu) { - Genode::warning("failed to lookup CPU"); - user_arg_0(-2); - return; - } + Cpu & cpu = cpu_pool()->cpu(user_arg_2()); user_arg_0(0); Thread * const thread = (Thread*) user_arg_1(); - assert(thread->_state == AWAITS_START) + assert(thread->_state == AWAITS_START); - thread->affinity(cpu); + thread->affinity(&cpu); /* join protection domain */ thread->_pd = (Pd *) user_arg_3(); @@ -313,6 +332,29 @@ void Thread::_call_yield_thread() } +void Thread::_call_delete_thread() +{ + Thread * to_delete = reinterpret_cast(user_arg_1()); + + /** + * Delete a thread immediately if it has no cpu assigned yet, + * or it is assigned to this cpu, or the assigned cpu did not scheduled it. + */ + if (!to_delete->_cpu || + (to_delete->_cpu->id() == Cpu::executing_id() || + &to_delete->_cpu->scheduled_job() != to_delete)) { + _call_delete(); + return; + } + + /** + * Construct a cross-cpu work item and send an IPI + */ + _destroy.construct(*this, *to_delete); + to_delete->_cpu->trigger_ip_interrupt(); +} + + void Thread::_call_await_request_msg() { if (Ipc_node::await_request(user_arg_1())) { @@ -557,6 +599,20 @@ void Thread::_call_delete_cap() } +void Kernel::Thread::_call_update_pd() +{ + Pd * const pd = (Pd *) user_arg_1(); + unsigned cnt = 0; + + cpu_pool()->for_each_cpu([&] (Cpu & cpu) { + /* if a cpu needs to update increase the counter */ + if (pd->update(cpu)) cnt++; }); + + /* insert the work item in the list if there are outstanding cpus */ + if (cnt) _pd_update.construct(*this, *pd, cnt); +} + + void Thread::_call() { try { @@ -597,7 +653,7 @@ void Thread::_call() case call_id_new_thread(): _call_new_thread(); return; case call_id_new_core_thread(): _call_new_core_thread(); return; case call_id_thread_quota(): _call_thread_quota(); return; - case call_id_delete_thread(): _call_delete(); return; + case call_id_delete_thread(): _call_delete_thread(); return; case call_id_start_thread(): _call_start_thread(); return; case call_id_resume_thread(): _call_resume_thread(); return; case call_id_cancel_thread_blocking(): _call_cancel_thread_blocking(); return; @@ -695,7 +751,7 @@ Core_thread::Core_thread() regs->sp = (addr_t)&__initial_stack_base[0] + DEFAULT_STACK_SIZE; regs->ip = (addr_t)&_core_start; - affinity(cpu_pool()->primary_cpu()); + affinity(&cpu_pool()->primary_cpu()); _utcb = utcb; Thread::_pd = core_pd(); _become_active(); diff --git a/repos/base-hw/src/core/kernel/thread.h b/repos/base-hw/src/core/kernel/thread.h index 2797eb6ef..0583a2500 100644 --- a/repos/base-hw/src/core/kernel/thread.h +++ b/repos/base-hw/src/core/kernel/thread.h @@ -14,13 +14,16 @@ #ifndef _CORE__KERNEL__THREAD_H_ #define _CORE__KERNEL__THREAD_H_ +#include +#include + /* core includes */ #include +#include +#include #include #include -#include #include -#include namespace Kernel { @@ -47,7 +50,7 @@ struct Kernel::Thread_fault */ class Kernel::Thread : - public Kernel::Object, public Cpu_job, public Cpu_domain_update, + public Kernel::Object, public Cpu_job, public Ipc_node, public Signal_context_killer, public Signal_handler, private Timeout { @@ -59,6 +62,47 @@ class Kernel::Thread Thread(Thread const &); Thread &operator = (Thread const &); + /** + * An update of page-table entries that requires architecture-wise + * maintainance operations, e.g., a TLB invalidation needs + * cross-cpu synchronization + */ + struct Pd_update : Inter_processor_work + { + Thread & caller; /* the caller gets blocked until all finished */ + Pd & pd; /* the corresponding pd */ + unsigned cnt; /* count of cpus left */ + + Pd_update(Thread & caller, Pd & pd, unsigned cnt); + + /************************************ + ** Inter_processor_work interface ** + ************************************/ + + void execute(); + }; + + /** + * The destruction of a thread still active on another cpu + * needs cross-cpu synchronization + */ + struct Destroy : Inter_processor_work + { + Thread & caller; /* the caller gets blocked till the end */ + Thread & thread_to_destroy; /* thread to be destroyed */ + + Destroy(Thread & caller, Thread & to_destroy); + + /************************************ + ** Inter_processor_work interface ** + ************************************/ + + void execute(); + }; + + friend void Pd_update::execute(); + friend void Destroy::execute(); + protected: enum { START_VERBOSE = 0 }; @@ -84,6 +128,9 @@ class Kernel::Thread bool _cancel_next_await_signal = false; bool const _core = false; + Genode::Constructible _pd_update {}; + Genode::Constructible _destroy {}; + /** * Notice that another thread yielded the CPU to this thread */ @@ -160,6 +207,7 @@ class Kernel::Thread void _call_cancel_thread_blocking(); void _call_restart_thread(); void _call_yield_thread(); + void _call_delete_thread(); void _call_await_request_msg(); void _call_send_request_msg(); void _call_send_reply_msg(); @@ -231,13 +279,6 @@ class Kernel::Thread void _await_request_succeeded(); void _await_request_failed(); - - /*********************** - ** Cpu_domain_update ** - ***********************/ - - void _cpu_domain_update_unblocks() { _restart(); } - public: Genode::Align_at regs; diff --git a/repos/base-hw/src/core/spec/arm/cpu_support.h b/repos/base-hw/src/core/spec/arm/cpu_support.h index 499d1110a..61ee26474 100644 --- a/repos/base-hw/src/core/spec/arm/cpu_support.h +++ b/repos/base-hw/src/core/spec/arm/cpu_support.h @@ -132,6 +132,15 @@ struct Genode::Arm_cpu : public Hw::Arm_cpu for (; base < top; base += line_size) { Icimvau::write(base); } } + /** + * Invalidate TLB regarding the given address space id + */ + static void invalidate_tlb(unsigned asid) + { + if (asid) Tlbiasid::write(asid); + else Tlbiall::write(0); + } + void switch_to(Context&, Mmu_context & o) { if (o.cidr == 0) return; diff --git a/repos/base-hw/src/core/spec/arm/kernel/cpu.cc b/repos/base-hw/src/core/spec/arm/kernel/cpu.cc index 653b3f52a..cbaa57f0e 100644 --- a/repos/base-hw/src/core/spec/arm/kernel/cpu.cc +++ b/repos/base-hw/src/core/spec/arm/kernel/cpu.cc @@ -14,25 +14,13 @@ /* core includes */ #include -#include #include -#include -void Kernel::Cpu::init(Kernel::Pic &pic) +void Kernel::Cpu::_arch_init() { /* enable performance counter */ perf_counter()->enable(); /* enable timer interrupt */ - pic.unmask(_timer.interrupt_id(), id()); -} - - -void Kernel::Cpu_domain_update::_domain_update() -{ - /* flush TLB by ASID */ - if (_domain_id) - Cpu::Tlbiasid::write(_domain_id); - else - Cpu::Tlbiall::write(0); + _pic.unmask(_timer.interrupt_id(), id()); } diff --git a/repos/base-hw/src/core/spec/arm/kernel/pd.cc b/repos/base-hw/src/core/spec/arm/kernel/pd.cc new file mode 100644 index 000000000..65d68fa88 --- /dev/null +++ b/repos/base-hw/src/core/spec/arm/kernel/pd.cc @@ -0,0 +1,29 @@ +/* + * \brief Kernel PD object implementations specific to ARM + * \author Stefan Kalkowski + * \date 2018-11-22 + */ + +/* + * Copyright (C) 2018 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include +#include + +bool Kernel::Pd::update(Cpu & cpu) +{ + /* invalidate the TLB on the local CPU only */ + if (cpu.id() == Cpu::executing_id()) { + Cpu::invalidate_tlb(mmu_regs.id()); + } + + /* + * on all SMP ARM platforms we support the TLB can be maintained + * cross-cpu coherently + */ + return false; +} diff --git a/repos/base-hw/src/core/spec/arm/kernel/thread.cc b/repos/base-hw/src/core/spec/arm/kernel/thread.cc index 7cbab5c98..8473cc5f5 100644 --- a/repos/base-hw/src/core/spec/arm/kernel/thread.cc +++ b/repos/base-hw/src/core/spec/arm/kernel/thread.cc @@ -54,7 +54,7 @@ void Thread::exception(Cpu & cpu) void Kernel::Thread::_call_update_data_region() { - Cpu * const cpu = cpu_pool()->cpu(Cpu::executing_id()); + Cpu & cpu = cpu_pool()->cpu(Cpu::executing_id()); /* * FIXME: If the caller is not a core thread, the kernel operates in a @@ -66,19 +66,19 @@ void Kernel::Thread::_call_update_data_region() * until then we apply operations to caches as a whole instead. */ if (!_core) { - cpu->clean_invalidate_data_cache(); + cpu.clean_invalidate_data_cache(); return; } auto base = (addr_t)user_arg_1(); auto const size = (size_t)user_arg_2(); - cpu->clean_invalidate_data_cache_by_virt_region(base, size); - cpu->invalidate_instr_cache(); + cpu.clean_invalidate_data_cache_by_virt_region(base, size); + cpu.invalidate_instr_cache(); } void Kernel::Thread::_call_update_instr_region() { - Cpu * const cpu = cpu_pool()->cpu(Cpu::executing_id()); + Cpu & cpu = cpu_pool()->cpu(Cpu::executing_id()); /* * FIXME: If the caller is not a core thread, the kernel operates in a @@ -90,18 +90,25 @@ void Kernel::Thread::_call_update_instr_region() * until then we apply operations to caches as a whole instead. */ if (!_core) { - cpu->clean_invalidate_data_cache(); - cpu->invalidate_instr_cache(); + cpu.clean_invalidate_data_cache(); + cpu.invalidate_instr_cache(); return; } auto base = (addr_t)user_arg_1(); auto const size = (size_t)user_arg_2(); - cpu->clean_invalidate_data_cache_by_virt_region(base, size); - cpu->invalidate_instr_cache_by_virt_region(base, size); + cpu.clean_invalidate_data_cache_by_virt_region(base, size); + cpu.invalidate_instr_cache_by_virt_region(base, size); } -extern void * kernel_stack; +/** + * on ARM with multiprocessing extensions, maintainance operations on TLB, + * and caches typically work coherently across CPUs when using the correct + * coprocessor registers (there might be ARM SoCs where this is not valid, + * with several shareability domains, but until now we do not support them) + */ +void Kernel::Thread::Pd_update::execute() { }; + void Thread::proceed(Cpu & cpu) { diff --git a/repos/base-hw/src/core/spec/arm/kernel/thread_update_pd.cc b/repos/base-hw/src/core/spec/arm/kernel/thread_update_pd.cc deleted file mode 100644 index 291974650..000000000 --- a/repos/base-hw/src/core/spec/arm/kernel/thread_update_pd.cc +++ /dev/null @@ -1,28 +0,0 @@ -/* - * \brief ARM non-SMP specific kernel thread implementations - * \author Stefan Kalkowski - * \date 2015-12-20 - */ - -/* - * Copyright (C) 2015-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ -/* core includes */ -#include -#include -#include - -void Kernel::Thread::_call_update_pd() -{ - Pd * const pd = (Pd *) user_arg_1(); - Cpu * const cpu = cpu_pool()->cpu(Cpu::executing_id()); - cpu->invalidate_instr_cache(); - cpu->clean_invalidate_data_cache(); - if (pd->mmu_regs.id()) - Cpu::Tlbiasid::write(pd->mmu_regs.id()); /* flush TLB by ASID */ - else - Cpu::Tlbiall::write(0); -} diff --git a/repos/base-hw/src/core/spec/arm/smp/kernel/cpu.cc b/repos/base-hw/src/core/spec/arm/smp/kernel/cpu.cc deleted file mode 100644 index 26deff7b0..000000000 --- a/repos/base-hw/src/core/spec/arm/smp/kernel/cpu.cc +++ /dev/null @@ -1,35 +0,0 @@ -/* - * \brief Cpu class implementation specific to ARM SMP - * \author Stefan Kalkowski - * \date 2015-12-09 - */ - -/* - * Copyright (C) 2015-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -/* core includes */ -#include -#include -#include - -/* base-internal includes */ -#include - - -/* spin-lock used to synchronize kernel access of different cpus */ -Kernel::Lock & Kernel::data_lock() { - return *unmanaged_singleton(); } - - -void Kernel::Cpu_domain_update::_domain_update() -{ - /* flush TLB by ASID */ - if (_domain_id) - Cpu::Tlbiasid::write(_domain_id); - else - Cpu::Tlbiall::write(0); -} diff --git a/repos/base-hw/src/core/spec/arm/smp/kernel/thread_update_pd.cc b/repos/base-hw/src/core/spec/arm/smp/kernel/thread_update_pd.cc deleted file mode 100644 index a4943d9ac..000000000 --- a/repos/base-hw/src/core/spec/arm/smp/kernel/thread_update_pd.cc +++ /dev/null @@ -1,23 +0,0 @@ -/* - * \brief ARM SMP specific kernel thread implementations - * \author Stefan Kalkowski - * \date 2015-12-20 - */ - -/* - * Copyright (C) 2015-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -/* core includes */ -#include -#include - -void Kernel::Thread::_call_update_pd() -{ - Pd * const pd = (Pd *) user_arg_1(); - if (Cpu_domain_update::_do_global(pd->mmu_regs.id())) { - _become_inactive(AWAITS_RESTART); } -} diff --git a/repos/base-hw/src/core/spec/arm_v7/cpu_support.h b/repos/base-hw/src/core/spec/arm_v7/cpu_support.h index c7ecc9970..cf7aa4e8a 100644 --- a/repos/base-hw/src/core/spec/arm_v7/cpu_support.h +++ b/repos/base-hw/src/core/spec/arm_v7/cpu_support.h @@ -22,6 +22,15 @@ namespace Genode { struct Arm_v7_cpu; } struct Genode::Arm_v7_cpu : Arm_cpu { + /** + * Returns whether this cpu implements the multiprocessor extensions + */ + static bool multi_processor() + { + static bool mp = Mpidr::Me::get(Mpidr::read()); + return mp; + } + /** * Write back dirty lines of inner data cache and invalidate all */ @@ -31,6 +40,17 @@ struct Genode::Arm_v7_cpu : Arm_cpu * Invalidate all lines of the inner data cache */ static void invalidate_inner_data_cache(); + + /** + * Invalidate TLB for given address space id + */ + static void invalidate_tlb(unsigned asid) + { + if (multi_processor()) { + if (asid) Tlbiasidis::write(asid); + else Tlbiallis::write(0); + } else Arm_cpu::invalidate_tlb(asid); + } }; #endif /* _CORE__SPEC__ARM_V7__CPU_SUPPORT_H_ */ diff --git a/repos/base-hw/src/core/spec/arm_v7/trustzone/kernel/vm.cc b/repos/base-hw/src/core/spec/arm_v7/trustzone/kernel/vm.cc index af1a3725e..7edc55ba9 100644 --- a/repos/base-hw/src/core/spec/arm_v7/trustzone/kernel/vm.cc +++ b/repos/base-hw/src/core/spec/arm_v7/trustzone/kernel/vm.cc @@ -25,7 +25,7 @@ Kernel::Vm::Vm(void * const state, : Cpu_job(Cpu_priority::MIN, 0), _state((Genode::Vm_state * const)state), _context(context), _table(0) { - affinity(cpu_pool()->primary_cpu()); } + affinity(&cpu_pool()->primary_cpu()); } Kernel::Vm::~Vm() {} diff --git a/repos/base-hw/src/core/spec/arm_v7/virtualization/kernel/vm.cc b/repos/base-hw/src/core/spec/arm_v7/virtualization/kernel/vm.cc index 9162ab6c9..770fc17a7 100644 --- a/repos/base-hw/src/core/spec/arm_v7/virtualization/kernel/vm.cc +++ b/repos/base-hw/src/core/spec/arm_v7/virtualization/kernel/vm.cc @@ -58,7 +58,7 @@ struct Kernel::Vm_irq : Kernel::Irq { Vm_irq(unsigned const irq) : - Kernel::Irq(irq, cpu_pool()->executing_cpu()->irq_pool()) + Kernel::Irq(irq, cpu_pool()->executing_cpu().irq_pool()) { } /** @@ -66,7 +66,7 @@ struct Kernel::Vm_irq : Kernel::Irq */ void occurred() { - Cpu_job & job = cpu_pool()->executing_cpu()->scheduled_job(); + Cpu_job & job = cpu_pool()->executing_cpu().scheduled_job(); Vm *vm = dynamic_cast(&job); if (!vm) Genode::error("VM timer interrupt while VM is not runnning!"); @@ -213,7 +213,7 @@ Kernel::Vm::Vm(void * const state, _context(context), _table(table) { - affinity(cpu_pool()->primary_cpu()); + affinity(&cpu_pool()->primary_cpu()); Virtual_pic::pic().irq.enable(); vt_host_context.sp = _cpu->stack_start(); diff --git a/repos/base-hw/src/core/spec/cortex_a15/kernel/cpu.cc b/repos/base-hw/src/core/spec/cortex_a15/kernel/cpu.cc deleted file mode 100644 index e25482be7..000000000 --- a/repos/base-hw/src/core/spec/cortex_a15/kernel/cpu.cc +++ /dev/null @@ -1,33 +0,0 @@ -/* - * \brief Cpu class implementation specific to Cortex A15 SMP - * \author Stefan Kalkowski - * \date 2015-12-09 - */ - -/* - * Copyright (C) 2015-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -/* core includes */ -#include -#include -#include - -/* base-hw includes */ -#include - -using namespace Kernel; - -void Kernel::Cpu::init(Kernel::Pic &pic) -{ - Lock::Guard guard(data_lock()); - - /* enable performance counter */ - perf_counter()->enable(); - - /* enable timer interrupt */ - pic.unmask(_timer.interrupt_id(), id()); -} diff --git a/repos/base-hw/src/core/spec/cortex_a8/cpu.cc b/repos/base-hw/src/core/spec/cortex_a8/cpu.cc index 833c39ff0..643100e75 100644 --- a/repos/base-hw/src/core/spec/cortex_a8/cpu.cc +++ b/repos/base-hw/src/core/spec/cortex_a8/cpu.cc @@ -17,8 +17,8 @@ #include -void Genode::Cpu::translation_added(Genode::addr_t const addr, - Genode::size_t const size) +void Genode::Cpu::translation_added(Genode::addr_t const, + Genode::size_t const) { using namespace Kernel; @@ -29,9 +29,5 @@ void Genode::Cpu::translation_added(Genode::addr_t const addr, * page table entry is added. We only do this as core as the kernel * adds translations solely before MMU and caches are enabled. */ - if (is_user()) update_data_region(addr, size); - else { - Cpu * const cpu = cpu_pool()->cpu(Cpu::executing_id()); - cpu->clean_invalidate_data_cache(); - } + Cpu::clean_invalidate_data_cache(); } diff --git a/repos/base-hw/src/core/spec/cortex_a9/kernel/cpu.cc b/repos/base-hw/src/core/spec/cortex_a9/kernel/cpu.cc deleted file mode 100644 index a02942bfe..000000000 --- a/repos/base-hw/src/core/spec/cortex_a9/kernel/cpu.cc +++ /dev/null @@ -1,35 +0,0 @@ -/* - * \brief Cpu class implementation specific to Cortex A9 SMP - * \author Stefan Kalkowski - * \date 2015-12-09 - */ - -/* - * Copyright (C) 2015-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -/* core includes */ -#include -#include -#include -#include -#include -#include -#include - - -void Kernel::Cpu::init(Kernel::Pic &pic) -{ - { - Lock::Guard guard(data_lock()); - - /* enable performance counter */ - perf_counter()->enable(); - - /* enable timer interrupt */ - pic.unmask(_timer.interrupt_id(), id()); - } -} diff --git a/repos/base-hw/src/core/spec/imx53/pic.h b/repos/base-hw/src/core/spec/imx53/pic.h index 647cd47c4..f21069f00 100644 --- a/repos/base-hw/src/core/spec/imx53/pic.h +++ b/repos/base-hw/src/core/spec/imx53/pic.h @@ -37,8 +37,6 @@ class Genode::Pic : public Hw::Pic void trigger(unsigned const i) { write(Swint::Intid::bits(i)); } - void trigger_ip_interrupt(unsigned) { } - bool secure(unsigned i) { return !read(i); } diff --git a/repos/base-hw/src/core/spec/riscv/kernel/cpu.cc b/repos/base-hw/src/core/spec/riscv/kernel/cpu.cc index 92c5b9928..f06ad0c93 100644 --- a/repos/base-hw/src/core/spec/riscv/kernel/cpu.cc +++ b/repos/base-hw/src/core/spec/riscv/kernel/cpu.cc @@ -15,5 +15,5 @@ #include #include -void Kernel::Cpu::init(Kernel::Pic &) { +void Kernel::Cpu::_arch_init() { Stvec::write(Hw::Mm::supervisor_exception_vector().base); } diff --git a/repos/base-hw/src/core/spec/riscv/kernel/pd.cc b/repos/base-hw/src/core/spec/riscv/kernel/pd.cc new file mode 100644 index 000000000..ea33568da --- /dev/null +++ b/repos/base-hw/src/core/spec/riscv/kernel/pd.cc @@ -0,0 +1,22 @@ +/* + * \brief Kernel pd object implementations for RiscV + * \author Stefan Kalkowski + * \date 2018-11-22 + */ + +/* + * Copyright (C) 2018 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include + +bool Kernel::Pd::update(Kernel::Cpu&) +{ + Genode::Cpu::sfence(); + return false; +} + + diff --git a/repos/base-hw/src/core/spec/riscv/kernel/thread.cc b/repos/base-hw/src/core/spec/riscv/kernel/thread.cc index 774581edf..aff818d82 100644 --- a/repos/base-hw/src/core/spec/riscv/kernel/thread.cc +++ b/repos/base-hw/src/core/spec/riscv/kernel/thread.cc @@ -18,6 +18,9 @@ using namespace Kernel; +void Thread::Pd_update::execute() {} + + void Thread::exception(Cpu&) { using Context = Genode::Cpu::Context; @@ -45,12 +48,6 @@ void Thread::exception(Cpu&) } -void Thread::_call_update_pd() -{ - Genode::Cpu::sfence(); -} - - void Thread::_call_update_data_region() { Genode::Cpu::sfence(); diff --git a/repos/base-hw/src/core/spec/smp/kernel/cpu.cc b/repos/base-hw/src/core/spec/smp/kernel/cpu.cc deleted file mode 100644 index 2591d9fa9..000000000 --- a/repos/base-hw/src/core/spec/smp/kernel/cpu.cc +++ /dev/null @@ -1,108 +0,0 @@ -/* - * \brief ARM with SMP support specific aspects of the kernel cpu objects - * \author Stefan Kalkowski - * \author Martin Stein - * \date 2016-01-07 - */ - -/* - * Copyright (C) 2016-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -/* core includes */ -#include - -namespace Kernel -{ - /** - * Lists all pending domain updates - */ - class Cpu_domain_update_list; -} - - -class Kernel::Cpu_domain_update_list -: public Double_list_typed -{ - public: - - /** - * Perform all pending domain updates on the executing CPU - */ - void do_each() { - for_each([] (Cpu_domain_update * const u) { u->_do(); }); } -}; - - -using namespace Kernel; - -/** - * Return singleton of the CPU domain-udpate list - */ -Cpu_domain_update_list & cpu_domain_update_list() { - return *unmanaged_singleton(); } - - - -void Cpu::Ipi::occurred() -{ - cpu_domain_update_list().do_each(); - pending = false; -} - - -void Cpu::Ipi::trigger(unsigned const cpu_id) -{ - if (pending) return; - - pic()->send_ipi(cpu_id); - pending = true; -} - - -Cpu::Ipi::Ipi(Irq::Pool &p) : Irq(Pic::IPI, p) { } - - -/*********************** - ** Cpu_domain_update ** - ***********************/ - -void Cpu_domain_update::_do() -{ - /* perform domain update locally and get pending bit */ - unsigned const id = Cpu::executing_id(); - if (!_pending[id]) return; - - _domain_update(); - _pending[id] = false; - - /* check wether there are still CPUs pending */ - for (unsigned i = 0; i < NR_OF_CPUS; i++) - if (_pending[i]) return; - - /* as no CPU is pending anymore, end the domain update */ - cpu_domain_update_list().remove(this); - _cpu_domain_update_unblocks(); -} - - -bool Cpu_domain_update::_do_global(unsigned const domain_id) -{ - /* perform locally and leave it at that if in uniprocessor mode */ - _domain_id = domain_id; - _domain_update(); - if (NR_OF_CPUS == 1) return false; - - /* inform other CPUs and block until they are done */ - cpu_domain_update_list().insert_tail(this); - unsigned const cpu_id = Cpu::executing_id(); - for (unsigned i = 0; i < NR_OF_CPUS; i++) { - if (i == cpu_id) continue; - _pending[i] = true; - cpu_pool()->cpu(i)->trigger_ip_interrupt(); - } - return true; -} diff --git a/repos/base-hw/src/core/spec/smp/kernel/kernel.cc b/repos/base-hw/src/core/spec/smp/kernel/kernel.cc deleted file mode 100644 index 199ef1b01..000000000 --- a/repos/base-hw/src/core/spec/smp/kernel/kernel.cc +++ /dev/null @@ -1,35 +0,0 @@ -/* - * \brief Kernel entrypoint for SMP systems - * \author Martin Stein - * \author Stefan Kalkowski - * \date 2011-10-20 - */ - -/* - * Copyright (C) 2011-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -/* core includes */ -#include -#include - - -extern "C" void kernel() -{ - using namespace Kernel; - - Cpu_job * new_job; - Cpu * cpu; - - { - Lock::Guard guard(data_lock()); - - cpu = cpu_pool()->cpu(Cpu::executing_id()); - new_job = &cpu->schedule(); - } - - new_job->proceed(*cpu); -} diff --git a/repos/base-hw/src/core/spec/x86_64/cpu.h b/repos/base-hw/src/core/spec/x86_64/cpu.h index 978cb8a63..051ab6e59 100644 --- a/repos/base-hw/src/core/spec/x86_64/cpu.h +++ b/repos/base-hw/src/core/spec/x86_64/cpu.h @@ -134,6 +134,12 @@ class Genode::Cpu : public Hw::X86_64_cpu void switch_to(Context & context, Mmu_context &mmu_context); static void mmu_fault(Context & regs, Kernel::Thread_fault & fault); + + /** + * Invalidate the whole TLB + */ + static void invalidate_tlb() { + Genode::Cpu::Cr3::write(Genode::Cpu::Cr3::read()); } }; #endif /* _CORE__SPEC__X86_64__CPU_H_ */ diff --git a/repos/base-hw/src/core/spec/x86_64/kernel/cpu.cc b/repos/base-hw/src/core/spec/x86_64/kernel/cpu.cc index a8aa0aafb..f3e6e23c3 100644 --- a/repos/base-hw/src/core/spec/x86_64/kernel/cpu.cc +++ b/repos/base-hw/src/core/spec/x86_64/kernel/cpu.cc @@ -15,10 +15,8 @@ /* core includes */ #include #include -#include - -void Kernel::Cpu::init(Pic &pic) +void Kernel::Cpu::_arch_init() { gdt.init((addr_t)&tss); Idt::init(); @@ -29,10 +27,6 @@ void Kernel::Cpu::init(Pic &pic) fpu().init(); /* enable timer interrupt */ - unsigned const cpu = Cpu::executing_id(); - pic.store_apic_id(cpu); - pic.unmask(_timer.interrupt_id(), cpu); + _pic.store_apic_id(id()); + _pic.unmask(_timer.interrupt_id(), id()); } - - -void Kernel::Cpu_domain_update::_domain_update() { } diff --git a/repos/base-hw/src/core/spec/x86_64/kernel/pd.cc b/repos/base-hw/src/core/spec/x86_64/kernel/pd.cc new file mode 100644 index 000000000..d98f17569 --- /dev/null +++ b/repos/base-hw/src/core/spec/x86_64/kernel/pd.cc @@ -0,0 +1,29 @@ +/* + * \brief X86-specific implementations for the kernel PD object + * \author Stefan Kalkowski + * \date 2018-11-22 + */ + +/* + * Copyright (C) 2018 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include +#include + + +bool Kernel::Pd::update(Cpu & cpu) +{ + /* on the current CPU invalidate the TLB */ + if (cpu.id() == Cpu::executing_id()) { + Cpu::invalidate_tlb(); + return false; + } + + /* for all other cpus send an IPI */ + cpu.trigger_ip_interrupt(); + return true; +} diff --git a/repos/base-hw/src/core/spec/x86_64/kernel/thread.cc b/repos/base-hw/src/core/spec/x86_64/kernel/thread.cc index fefa2090c..91b1bef8b 100644 --- a/repos/base-hw/src/core/spec/x86_64/kernel/thread.cc +++ b/repos/base-hw/src/core/spec/x86_64/kernel/thread.cc @@ -18,6 +18,18 @@ #include #include +void Kernel::Thread::Pd_update::execute() +{ + /* invalidate cpu-local TLB */ + Cpu::invalidate_tlb(); + + /* if this is the last cpu, wake up the caller thread */ + if (--cnt == 0) { + cpu_pool()->work_list().remove(&_le); + caller._restart(); + } +}; + void Kernel::Thread::_call_update_data_region() { } @@ -25,11 +37,6 @@ void Kernel::Thread::_call_update_data_region() { } void Kernel::Thread::_call_update_instr_region() { } -void Kernel::Thread::_call_update_pd() { - Genode::Cpu::Cr3::write(Genode::Cpu::Cr3::read()); -} - - void Kernel::Thread::proceed(Cpu & cpu) { cpu.switch_to(*regs, pd()->mmu_regs); diff --git a/repos/base-hw/src/core/spec/x86_64/muen/kernel/vm.cc b/repos/base-hw/src/core/spec/x86_64/muen/kernel/vm.cc index 00485eab9..99ab0ceba 100644 --- a/repos/base-hw/src/core/spec/x86_64/muen/kernel/vm.cc +++ b/repos/base-hw/src/core/spec/x86_64/muen/kernel/vm.cc @@ -27,7 +27,7 @@ Kernel::Vm::Vm(void * const state, Kernel::Signal_context * const context, _context(context), _table(nullptr) { - affinity(cpu_pool()->primary_cpu()); + affinity(&cpu_pool()->primary_cpu()); } diff --git a/repos/base-hw/src/core/spec/x86_64/muen/pic.h b/repos/base-hw/src/core/spec/x86_64/muen/pic.h index 580775d2c..a02940df7 100644 --- a/repos/base-hw/src/core/spec/x86_64/muen/pic.h +++ b/repos/base-hw/src/core/spec/x86_64/muen/pic.h @@ -61,7 +61,6 @@ class Genode::Pic void unmask(unsigned const, unsigned) { } void mask(unsigned const) { } bool is_ip_interrupt(unsigned, unsigned) { return false; } - void trigger_ip_interrupt(unsigned) { } void store_apic_id(unsigned const) { } private: diff --git a/repos/base-hw/src/core/spec/x86_64/smp/cpu.cc b/repos/base-hw/src/core/spec/x86_64/smp/cpu.cc deleted file mode 100644 index b370b7acc..000000000 --- a/repos/base-hw/src/core/spec/x86_64/smp/cpu.cc +++ /dev/null @@ -1,25 +0,0 @@ -/* - * \brief Cpu class implementation specific to SMP - * \author Stefan Kalkowski - * \date 2015-12-09 - */ - -/* - * Copyright (C) 2015-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -/* core includes */ -#include -#include -#include - -/* base-internal includes */ -#include - - -/* spin-lock used to synchronize kernel access of different cpus */ -Kernel::Lock & Kernel::data_lock() { - return *unmanaged_singleton(); } diff --git a/repos/base-hw/src/lib/hw/spec/arm/cpu.h b/repos/base-hw/src/lib/hw/spec/arm/cpu.h index 8be3ff916..122371b4f 100644 --- a/repos/base-hw/src/lib/hw/spec/arm/cpu.h +++ b/repos/base-hw/src/lib/hw/spec/arm/cpu.h @@ -193,9 +193,15 @@ struct Hw::Arm_cpu /* Invalidate entire unified TLB */ ARM_CP15_REGISTER_32BIT(Tlbiall, c8, c7, 0, 0); + /* Invalidate entire unified TLB (inner-shareable) */ + ARM_CP15_REGISTER_32BIT(Tlbiallis, c8, c3, 0, 0); + /* Invalidate unified TLB by ASID */ ARM_CP15_REGISTER_32BIT(Tlbiasid, c8, c7, 0, 2); + /* Invalidate unified TLB by ASID (inner-shareable) */ + ARM_CP15_REGISTER_32BIT(Tlbiasidis, c8, c3, 0, 2); + /* Memory Attribute Indirection Register 0 */ ARM_CP15_REGISTER_32BIT(Mair0, c10, c2, 0, 0, struct Attr0 : Bitfield<0, 8> { }; @@ -256,6 +262,12 @@ struct Hw::Arm_cpu static void clean_invalidate_data_cache(); static void invalidate_data_cache(); + + static inline void synchronization_barrier() + { + asm volatile("dsb\n" + "isb\n"); + } }; #endif /* _SRC__LIB__HW__SPEC__ARM__CPU_H_ */