diff options
Diffstat (limited to 'lib/locks/bakery')
| -rw-r--r-- | lib/locks/bakery/bakery_lock_coherent.c | 80 | ||||
| -rw-r--r-- | lib/locks/bakery/bakery_lock_normal.c | 133 |
2 files changed, 91 insertions, 122 deletions
diff --git a/lib/locks/bakery/bakery_lock_coherent.c b/lib/locks/bakery/bakery_lock_coherent.c index 5d538ce2..a857e035 100644 --- a/lib/locks/bakery/bakery_lock_coherent.c +++ b/lib/locks/bakery/bakery_lock_coherent.c @@ -1,31 +1,7 @@ /* - * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2015, ARM Limited and Contributors. All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of ARM nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ #include <arch_helpers.h> @@ -63,27 +39,15 @@ assert(entry < BAKERY_LOCK_MAX_CPUS); \ } while (0) -/* Convert a ticket to priority */ -#define PRIORITY(t, pos) (((t) << 8) | (pos)) - - -/* Initialize Bakery Lock to reset ownership and all ticket values */ -void bakery_lock_init(bakery_lock_t *bakery) -{ - assert(bakery); - - /* All ticket values need to be 0 */ - memset(bakery, 0, sizeof(*bakery)); - bakery->owner = NO_OWNER; -} - - /* Obtain a ticket for a given CPU */ static unsigned int bakery_get_ticket(bakery_lock_t *bakery, unsigned int me) { unsigned int my_ticket, their_ticket; unsigned int they; + /* Prevent recursive acquisition */ + assert(!bakery_ticket_number(bakery->lock_data[me])); + /* * Flag that we're busy getting our ticket. All CPUs are iterated in the * order of their ordinal position to decide the maximum ticket value @@ -95,9 +59,9 @@ static unsigned int bakery_get_ticket(bakery_lock_t *bakery, unsigned int me) * value, not the ticket value alone. */ my_ticket = 0; - bakery->entering[me] = 1; + bakery->lock_data[me] = make_bakery_data(CHOOSING_TICKET, my_ticket); for (they = 0; they < BAKERY_LOCK_MAX_CPUS; they++) { - their_ticket = bakery->number[they]; + their_ticket = bakery_ticket_number(bakery->lock_data[they]); if (their_ticket > my_ticket) my_ticket = their_ticket; } @@ -107,8 +71,7 @@ static unsigned int bakery_get_ticket(bakery_lock_t *bakery, unsigned int me) * finish calculating our ticket value that we're done */ ++my_ticket; - bakery->number[me] = my_ticket; - bakery->entering[me] = 0; + bakery->lock_data[me] = make_bakery_data(CHOSEN_TICKET, my_ticket); return my_ticket; } @@ -129,14 +92,12 @@ void bakery_lock_get(bakery_lock_t *bakery) { unsigned int they, me; unsigned int my_ticket, my_prio, their_ticket; + unsigned int their_bakery_data; - me = platform_get_core_pos(read_mpidr_el1()); + me = plat_my_core_pos(); assert_bakery_entry_valid(me, bakery); - /* Prevent recursive acquisition */ - assert(bakery->owner != me); - /* Get a ticket */ my_ticket = bakery_get_ticket(bakery, me); @@ -150,14 +111,15 @@ void bakery_lock_get(bakery_lock_t *bakery) continue; /* Wait for the contender to get their ticket */ - while (bakery->entering[they]) - ; + do { + their_bakery_data = bakery->lock_data[they]; + } while (bakery_is_choosing(their_bakery_data)); /* * If the other party is a contender, they'll have non-zero * (valid) ticket value. If they do, compare priorities */ - their_ticket = bakery->number[they]; + their_ticket = bakery_ticket_number(their_bakery_data); if (their_ticket && (PRIORITY(their_ticket, they) < my_prio)) { /* * They have higher priority (lower value). Wait for @@ -167,29 +129,27 @@ void bakery_lock_get(bakery_lock_t *bakery) */ do { wfe(); - } while (their_ticket == bakery->number[they]); + } while (their_ticket == + bakery_ticket_number(bakery->lock_data[they])); } } - /* Lock acquired */ - bakery->owner = me; } /* Release the lock and signal contenders */ void bakery_lock_release(bakery_lock_t *bakery) { - unsigned int me = platform_get_core_pos(read_mpidr_el1()); + unsigned int me = plat_my_core_pos(); assert_bakery_entry_valid(me, bakery); - assert(bakery->owner == me); + assert(bakery_ticket_number(bakery->lock_data[me])); /* - * Release lock by resetting ownership and ticket. Then signal other + * Release lock by resetting ticket. Then signal other * waiting contenders */ - bakery->owner = NO_OWNER; - bakery->number[me] = 0; + bakery->lock_data[me] = 0; dsb(); sev(); } diff --git a/lib/locks/bakery/bakery_lock_normal.c b/lib/locks/bakery/bakery_lock_normal.c index a325fd4f..8f59215e 100644 --- a/lib/locks/bakery/bakery_lock_normal.c +++ b/lib/locks/bakery/bakery_lock_normal.c @@ -1,31 +1,7 @@ /* - * Copyright (c) 2015, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2015-2017, ARM Limited and Contributors. All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of ARM nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ #include <arch_helpers.h> @@ -56,35 +32,57 @@ * accesses regardless of status of address translation. */ -/* Convert a ticket to priority */ -#define PRIORITY(t, pos) (((t) << 8) | (pos)) - -#define CHOOSING_TICKET 0x1 -#define CHOOSING_DONE 0x0 - -#define bakery_is_choosing(info) (info & 0x1) -#define bakery_ticket_number(info) ((info >> 1) & 0x7FFF) -#define make_bakery_data(choosing, number) \ - (((choosing & 0x1) | (number << 1)) & 0xFFFF) - -/* This macro assumes that the bakery_info array is located at the offset specified */ -#define get_my_bakery_info(offset, id) \ - (((bakery_info_t *) (((uint8_t *)_cpu_data()) + offset)) + id) +#ifdef PLAT_PERCPU_BAKERY_LOCK_SIZE +/* + * Verify that the platform defined value for the per-cpu space for bakery locks is + * a multiple of the cache line size, to prevent multiple CPUs writing to the same + * bakery lock cache line + * + * Using this value, if provided, rather than the linker generated value results in + * more efficient code + */ +CASSERT((PLAT_PERCPU_BAKERY_LOCK_SIZE & (CACHE_WRITEBACK_GRANULE - 1)) == 0, \ + PLAT_PERCPU_BAKERY_LOCK_SIZE_not_cacheline_multiple); +#define PERCPU_BAKERY_LOCK_SIZE (PLAT_PERCPU_BAKERY_LOCK_SIZE) +#else +/* + * Use the linker defined symbol which has evaluated the size reqiurement. + * This is not as efficient as using a platform defined constant + */ +extern void *__PERCPU_BAKERY_LOCK_SIZE__; +#define PERCPU_BAKERY_LOCK_SIZE ((uintptr_t)&__PERCPU_BAKERY_LOCK_SIZE__) +#endif -#define get_bakery_info_by_index(offset, id, ix) \ - (((bakery_info_t *) (((uint8_t *)_cpu_data_by_index(ix)) + offset)) + id) +#define get_bakery_info(cpu_ix, lock) \ + (bakery_info_t *)((uintptr_t)lock + cpu_ix * PERCPU_BAKERY_LOCK_SIZE) #define write_cache_op(addr, cached) \ do { \ - (cached ? dccvac((uint64_t)addr) :\ - dcivac((uint64_t)addr));\ + (cached ? dccvac((uintptr_t)addr) :\ + dcivac((uintptr_t)addr));\ dsbish();\ } while (0) #define read_cache_op(addr, cached) if (cached) \ - dccivac((uint64_t)addr) + dccivac((uintptr_t)addr) -static unsigned int bakery_get_ticket(int id, unsigned int offset, +/* Helper function to check if the lock is acquired */ +static inline int is_lock_acquired(const bakery_info_t *my_bakery_info, + int is_cached) +{ + /* + * Even though lock data is updated only by the owning cpu and + * appropriate cache maintenance operations are performed, + * if the previous update was done when the cpu was not participating + * in coherency, then there is a chance that cache maintenance + * operations were not propagated to all the caches in the system. + * Hence do a `read_cache_op()` prior to read. + */ + read_cache_op(my_bakery_info, is_cached); + return !!(bakery_ticket_number(my_bakery_info->lock_data)); +} + +static unsigned int bakery_get_ticket(bakery_lock_t *lock, unsigned int me, int is_cached) { unsigned int my_ticket, their_ticket; @@ -95,9 +93,12 @@ static unsigned int bakery_get_ticket(int id, unsigned int offset, * Obtain a reference to the bakery information for this cpu and ensure * it is not NULL. */ - my_bakery_info = get_my_bakery_info(offset, id); + my_bakery_info = get_bakery_info(me, lock); assert(my_bakery_info); + /* Prevent recursive acquisition.*/ + assert(!is_lock_acquired(my_bakery_info, is_cached)); + /* * Tell other contenders that we are through the bakery doorway i.e. * going to allocate a ticket for this cpu. @@ -119,7 +120,7 @@ static unsigned int bakery_get_ticket(int id, unsigned int offset, * Get a reference to the other contender's bakery info and * ensure that a stale copy is not read. */ - their_bakery_info = get_bakery_info_by_index(offset, id, they); + their_bakery_info = get_bakery_info(they, lock); assert(their_bakery_info); read_cache_op(their_bakery_info, is_cached); @@ -138,26 +139,29 @@ static unsigned int bakery_get_ticket(int id, unsigned int offset, * finish calculating our ticket value that we're done */ ++my_ticket; - my_bakery_info->lock_data = make_bakery_data(CHOOSING_DONE, my_ticket); + my_bakery_info->lock_data = make_bakery_data(CHOSEN_TICKET, my_ticket); write_cache_op(my_bakery_info, is_cached); return my_ticket; } -void bakery_lock_get(unsigned int id, unsigned int offset) +void bakery_lock_get(bakery_lock_t *lock) { unsigned int they, me, is_cached; unsigned int my_ticket, my_prio, their_ticket; bakery_info_t *their_bakery_info; - uint16_t their_bakery_data; - - me = platform_get_core_pos(read_mpidr_el1()); + unsigned int their_bakery_data; + me = plat_my_core_pos(); +#ifdef AARCH32 + is_cached = read_sctlr() & SCTLR_C_BIT; +#else is_cached = read_sctlr_el3() & SCTLR_C_BIT; +#endif /* Get a ticket */ - my_ticket = bakery_get_ticket(id, offset, me, is_cached); + my_ticket = bakery_get_ticket(lock, me, is_cached); /* * Now that we got our ticket, compute our priority value, then compare @@ -172,17 +176,14 @@ void bakery_lock_get(unsigned int id, unsigned int offset) * Get a reference to the other contender's bakery info and * ensure that a stale copy is not read. */ - their_bakery_info = get_bakery_info_by_index(offset, id, they); + their_bakery_info = get_bakery_info(they, lock); assert(their_bakery_info); - read_cache_op(their_bakery_info, is_cached); - - their_bakery_data = their_bakery_info->lock_data; /* Wait for the contender to get their ticket */ - while (bakery_is_choosing(their_bakery_data)) { + do { read_cache_op(their_bakery_info, is_cached); their_bakery_data = their_bakery_info->lock_data; - } + } while (bakery_is_choosing(their_bakery_data)); /* * If the other party is a contender, they'll have non-zero @@ -203,14 +204,22 @@ void bakery_lock_get(unsigned int id, unsigned int offset) == bakery_ticket_number(their_bakery_info->lock_data)); } } + /* Lock acquired */ } -void bakery_lock_release(unsigned int id, unsigned int offset) +void bakery_lock_release(bakery_lock_t *lock) { bakery_info_t *my_bakery_info; +#ifdef AARCH32 + unsigned int is_cached = read_sctlr() & SCTLR_C_BIT; +#else unsigned int is_cached = read_sctlr_el3() & SCTLR_C_BIT; +#endif + + my_bakery_info = get_bakery_info(plat_my_core_pos(), lock); + + assert(is_lock_acquired(my_bakery_info, is_cached)); - my_bakery_info = get_my_bakery_info(offset, id); my_bakery_info->lock_data = 0; write_cache_op(my_bakery_info, is_cached); sev(); |