path: root/src/arm/linux/init.c

#include <stdint.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>

#include <cpuinfo.h>
#include <arm/linux/api.h>
#if defined(__ANDROID__)
	#include <arm/android/api.h>
#endif
#include <gpu/api.h>
#include <arm/api.h>
#include <arm/midr.h>
#include <linux/api.h>
#include <api.h>
#include <log.h>


struct cpuinfo_arm_isa cpuinfo_isa = { 0 };

static struct cpuinfo_package package = { { 0 } };

static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) {
	return (bitfield & mask) == mask;
}

static inline uint32_t min(uint32_t a, uint32_t b) {
	return a < b ? a : b;
}

static inline int cmp(uint32_t a, uint32_t b) {
	return (a > b) - (a < b);
}

static bool cluster_siblings_parser(
	uint32_t processor, uint32_t siblings_start, uint32_t siblings_end,
	struct cpuinfo_arm_linux_processor* processors)
{
	processors[processor].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;
	uint32_t package_leader_id = processors[processor].package_leader_id;

	for (uint32_t sibling = siblings_start; sibling < siblings_end; sibling++) {
		if (!bitmask_all(processors[sibling].flags, CPUINFO_LINUX_MASK_USABLE)) {
			cpuinfo_log_info("invalid processor %"PRIu32" reported as a sibling for processor %"PRIu32,
				sibling, processor);
			continue;
		}

		const uint32_t sibling_package_leader_id = processors[sibling].package_leader_id;
		if (sibling_package_leader_id < package_leader_id) {
			package_leader_id = sibling_package_leader_id;
		}

		processors[sibling].package_leader_id = package_leader_id;
		processors[sibling].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;
	}

	processors[processor].package_leader_id = package_leader_id;

	return true;
}

static int cmp_arm_linux_processor(const void* ptr_a, const void* ptr_b) {
	const struct cpuinfo_arm_linux_processor* processor_a = (const struct cpuinfo_arm_linux_processor*) ptr_a;
	const struct cpuinfo_arm_linux_processor* processor_b = (const struct cpuinfo_arm_linux_processor*) ptr_b;

	/* Move usable processors towards the start of the array */
	const bool usable_a = bitmask_all(processor_a->flags, CPUINFO_LINUX_MASK_USABLE);
	const bool usable_b = bitmask_all(processor_b->flags, CPUINFO_LINUX_MASK_USABLE);
	if (usable_a != usable_b) {
		return (int) usable_b - (int) usable_a;
	}

	/* Compare based on core type (e.g. Cortex-A57 < Cortex-A53) */
	const uint32_t midr_a = processor_a->midr;
	const uint32_t midr_b = processor_b->midr;
	if (midr_a != midr_b) {
		const uint32_t score_a = midr_score_core(midr_a);
		const uint32_t score_b = midr_score_core(midr_b);
		if (score_a != score_b) {
			return score_a > score_b ? -1 : 1;
		}
	}

	/* Compare based on core frequency (e.g. 2.0 GHz < 1.2 GHz) */
	const uint32_t frequency_a = processor_a->max_frequency;
	const uint32_t frequency_b = processor_b->max_frequency;
	if (frequency_a != frequency_b) {
		return frequency_a > frequency_b ? -1 : 1;
	}

	/* Compare based on cluster leader id (i.e. cluster 1 < cluster 0) */
	const uint32_t cluster_a = processor_a->package_leader_id;
	const uint32_t cluster_b = processor_b->package_leader_id;
	if (cluster_a != cluster_b) {
		return cluster_a > cluster_b ? -1 : 1;
	}

	/* Compare based on system processor id (i.e. processor 0 < processor 1) */
	const uint32_t id_a = processor_a->system_processor_id;
	const uint32_t id_b = processor_b->system_processor_id;
	return cmp(id_a, id_b);
}

void cpuinfo_arm_linux_init(void) {
	struct cpuinfo_arm_linux_processor* arm_linux_processors = NULL;
	struct cpuinfo_processor* processors = NULL;
	struct cpuinfo_core* cores = NULL;
	const struct cpuinfo_processor** linux_cpu_to_processor_map = NULL;
	const struct cpuinfo_core** linux_cpu_to_core_map = NULL;
	struct cpuinfo_cache* l1i = NULL;
	struct cpuinfo_cache* l1d = NULL;
	struct cpuinfo_cache* l2 = NULL;

	const uint32_t max_processors_count = cpuinfo_linux_get_max_processors_count();
	cpuinfo_log_debug("system maximum processors count: %"PRIu32, max_processors_count);

	const uint32_t max_possible_processors_count = 1 +
		cpuinfo_linux_get_max_possible_processor(max_processors_count);
	cpuinfo_log_debug("maximum possible processors count: %"PRIu32, max_possible_processors_count);
	const uint32_t max_present_processors_count = 1 +
		cpuinfo_linux_get_max_possible_processor(max_processors_count);
	cpuinfo_log_debug("maximum present processors count: %"PRIu32, max_present_processors_count);

	const uint32_t arm_linux_processors_count = min(max_possible_processors_count, max_present_processors_count);
	arm_linux_processors = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_arm_linux_processor));
	if (arm_linux_processors == NULL) {
		cpuinfo_log_error(
			"failed to allocate %zu bytes for descriptions of %"PRIu32" ARM logical processors",
			arm_linux_processors_count * sizeof(struct cpuinfo_arm_linux_processor),
			arm_linux_processors_count);
		return;
	}

	cpuinfo_linux_detect_possible_processors(
		arm_linux_processors_count, &arm_linux_processors->flags,
		sizeof(struct cpuinfo_arm_linux_processor),
		CPUINFO_LINUX_FLAG_POSSIBLE);

	cpuinfo_linux_detect_present_processors(
		arm_linux_processors_count, &arm_linux_processors->flags,
		sizeof(struct cpuinfo_arm_linux_processor),
		CPUINFO_LINUX_FLAG_PRESENT);

#if defined(__ANDROID__)
	struct cpuinfo_android_properties android_properties;
	cpuinfo_arm_android_parse_properties(&android_properties);
#else
	char proc_cpuinfo_hardware[CPUINFO_HARDWARE_VALUE_MAX] = { 0 };
#endif

	if (!cpuinfo_arm_linux_parse_proc_cpuinfo(
#if defined(__ANDROID__)
			android_properties.proc_cpuinfo_hardware,
#else
			proc_cpuinfo_hardware,
#endif
			arm_linux_processors_count,
			arm_linux_processors)) {
		cpuinfo_log_error("failed to parse processor information from /proc/cpuinfo");
		return;
	}

	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE)) {
			cpuinfo_log_debug("parsed processor %"PRIu32" MIDR 0x%08"PRIx32,
				i, arm_linux_processors[i].midr);
		}
	}

	uint32_t usable_processors = 0;
	#if CPUINFO_ARCH_ARM
	uint32_t last_midr = 0, last_architecture_version = 0, last_architecture_flags = 0;
	#endif
	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		arm_linux_processors[i].system_processor_id = i;
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE)) {
			usable_processors += 1;

			if (!(arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR)) {
				/*
				 * Processor is in possible and present lists, but not reported in /proc/cpuinfo.
				 * This is fairly common: high-index processors can be not reported if they are offline.
				 */
				cpuinfo_log_info("processor %"PRIu32" is not listed in /proc/cpuinfo", i);
			}

			#if CPUINFO_ARCH_ARM
				if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_MIDR)) {
					last_midr = arm_linux_processors[i].midr;
				}
				if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ARCHITECTURE)) {
					last_architecture_version = arm_linux_processors[i].architecture_version;
					last_architecture_flags   = arm_linux_processors[i].architecture_flags;
				}
			#endif
		} else {
			/* Processor reported in /proc/cpuinfo, but not in possible and/or present lists: log and ignore */
			if (!(arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR)) {
				cpuinfo_log_warning("invalid processor %"PRIu32" reported in /proc/cpuinfo", i);
			}
		}
	}

	/* Detect ISA. If different processors report different ISA features, take the intersection. */
	uint32_t isa_features = 0, processors_with_features = 0;
	#if CPUINFO_ARCH_ARM
		uint32_t isa_features2 = 0;
	#endif
	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE | CPUINFO_ARM_LINUX_VALID_FEATURES)) {
			if (processors_with_features == 0) {
				isa_features = arm_linux_processors[i].features;
				#if CPUINFO_ARCH_ARM
					isa_features2 = arm_linux_processors[i].features2;
				#endif
			} else {
				isa_features &= arm_linux_processors[i].features;
				#if CPUINFO_ARCH_ARM
					isa_features2 &= arm_linux_processors[i].features2;
				#endif
			}
			processors_with_features += 1;
		}
	}
	#if CPUINFO_ARCH_ARM
		cpuinfo_arm_linux_decode_isa_from_proc_cpuinfo(
			isa_features, isa_features2,
			last_midr, last_architecture_version, last_architecture_flags,
			&cpuinfo_isa);
	#elif CPUINFO_ARCH_ARM64
		cpuinfo_arm64_linux_decode_isa_from_proc_cpuinfo(
			isa_features, &cpuinfo_isa);
	#endif

	/* Detect min/max frequency and package ID */
	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE)) {
			const uint32_t max_frequency = cpuinfo_linux_get_processor_max_frequency(i);
			if (max_frequency != 0) {
				arm_linux_processors[i].max_frequency = max_frequency;
				arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY;
			}

			const uint32_t min_frequency = cpuinfo_linux_get_processor_min_frequency(i);
			if (min_frequency != 0) {
				arm_linux_processors[i].min_frequency = min_frequency;
				arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY;
			}

			if (cpuinfo_linux_get_processor_package_id(i, &arm_linux_processors[i].package_id)) {
				arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_ID;
			}
		}
	}

	/* Initialize topology group IDs */
	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		arm_linux_processors[i].package_leader_id = i;
	}

	/* Propagate topology group IDs among siblings */
	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (!bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE)) {
			continue;
		}

		if (arm_linux_processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_ID) {
			cpuinfo_linux_detect_core_siblings(
				arm_linux_processors_count, i,
				(cpuinfo_siblings_callback) cluster_siblings_parser,
				arm_linux_processors);
		}
	}

	/* Propagate all cluster IDs */
	uint32_t clustered_processors = 0;
	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE | CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) {
			clustered_processors += 1;

			const uint32_t package_leader_id = arm_linux_processors[i].package_leader_id;
			if (package_leader_id < i) {
				arm_linux_processors[i].package_leader_id = arm_linux_processors[package_leader_id].package_leader_id;
			}

			cpuinfo_log_debug("processor %"PRIu32" clustered with processor %"PRIu32" as inferred from system siblings lists",
				i, arm_linux_processors[i].package_leader_id);
		}
	}

	if (clustered_processors != usable_processors) {
		/*
		 * Topology information about some or all logical processors may be unavailable, for the following reasons:
		 * - Linux kernel is too old, or configured without support for topology information in sysfs.
		 * - Core is offline, and Linux kernel is configured to not report topology for offline cores.
		 *
		 * In this case, we assign processors to clusters using two methods:
		 * - Try heuristic cluster configurations (e.g. 6-core SoC usually has 4+2 big.LITTLE configuration).
		 * - If heuristic failed, assign processors to core clusters in a sequential scan.
		 */
		if (!cpuinfo_arm_linux_detect_core_clusters_by_heuristic(usable_processors, arm_linux_processors_count, arm_linux_processors)) {
			cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan(arm_linux_processors_count, arm_linux_processors);
		}
	}

	cpuinfo_arm_linux_count_cluster_processors(arm_linux_processors_count, arm_linux_processors);

#if defined(__ANDROID__)
	const struct cpuinfo_arm_chipset chipset =
		cpuinfo_arm_android_decode_chipset(&android_properties, usable_processors, 0);
#else
	const struct cpuinfo_arm_chipset chipset = {
		.vendor = cpuinfo_arm_chipset_vendor_unknown,
		.series = cpuinfo_arm_chipset_series_unknown,
	};
#endif

	const uint32_t cluster_count = cpuinfo_arm_linux_detect_cluster_midr(
		&chipset,
		arm_linux_processors_count, usable_processors, arm_linux_processors);

	/* Initialize core vendor, uarch, MIDR, and frequency for every logical processor */
	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE)) {
			const uint32_t cluster_leader = arm_linux_processors[i].package_leader_id;
			if (cluster_leader == i) {
				/* Cluster leader: decode core vendor and uarch */
				cpuinfo_arm_decode_vendor_uarch(
				arm_linux_processors[cluster_leader].midr,
#if CPUINFO_ARCH_ARM
				!!(arm_linux_processors[cluster_leader].features & CPUINFO_ARM_LINUX_FEATURE_VFPV4),
#endif
				&arm_linux_processors[cluster_leader].vendor,
				&arm_linux_processors[cluster_leader].uarch);
			} else {
				/* Cluster non-leader: copy vendor, uarch, MIDR, and frequency from cluster leader */
				arm_linux_processors[i].flags |= arm_linux_processors[cluster_leader].flags &
					(CPUINFO_ARM_LINUX_VALID_MIDR | CPUINFO_LINUX_FLAG_MAX_FREQUENCY);
				arm_linux_processors[i].midr = arm_linux_processors[cluster_leader].midr;
				arm_linux_processors[i].vendor = arm_linux_processors[cluster_leader].vendor;
				arm_linux_processors[i].uarch = arm_linux_processors[cluster_leader].uarch;
				arm_linux_processors[i].max_frequency = arm_linux_processors[cluster_leader].max_frequency;
			}
		}
	}

	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE)) {
			cpuinfo_log_debug("post-analysis processor %"PRIu32": MIDR %08"PRIx32" frequency %"PRIu32,
				i, arm_linux_processors[i].midr, arm_linux_processors[i].max_frequency);
		}
	}

	qsort(arm_linux_processors, arm_linux_processors_count,
		sizeof(struct cpuinfo_arm_linux_processor), cmp_arm_linux_processor);

	for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
		if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_MASK_USABLE)) {
			cpuinfo_log_debug("post-sort processor %"PRIu32": system id %"PRIu32" MIDR %08"PRIx32" frequency %"PRIu32,
				i, arm_linux_processors[i].system_processor_id, arm_linux_processors[i].midr, arm_linux_processors[i].max_frequency);
		}
	}

	/*
	 * Assumptions:
	 * - No SMP (i.e. each core supports only one hardware thread).
	 * - Level 1 instruction and data caches are private to the core clusters.
	 * - Level 2 cache is shared between cores in the same cluster.
	 */
	cpuinfo_arm_chipset_to_string(&chipset, package.name);
	package.processor_count = package.core_count = usable_processors;

	processors = calloc(usable_processors, sizeof(struct cpuinfo_processor));
	if (processors == NULL) {
		cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors",
			usable_processors * sizeof(struct cpuinfo_processor), usable_processors);
		goto cleanup;
	}

	cores = calloc(usable_processors, sizeof(struct cpuinfo_core));
	if (cores == NULL) {
		cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores",
			usable_processors * sizeof(struct cpuinfo_core), usable_processors);
		goto cleanup;
	}

	linux_cpu_to_processor_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_processor*));
	if (linux_cpu_to_processor_map == NULL) {
		cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" logical processor mapping entries",
			arm_linux_processors_count * sizeof(struct cpuinfo_processor*), arm_linux_processors_count);
		goto cleanup;
	}

	linux_cpu_to_core_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_core*));
	if (linux_cpu_to_core_map == NULL) {
		cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" core mapping entries",
			arm_linux_processors_count * sizeof(struct cpuinfo_core*), arm_linux_processors_count);
		goto cleanup;
	}

	l1i = calloc(usable_processors, sizeof(struct cpuinfo_cache));
	if (l1i == NULL) {
		cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches",
			usable_processors * sizeof(struct cpuinfo_cache), usable_processors);
		goto cleanup;
	}

	l1d = calloc(usable_processors, sizeof(struct cpuinfo_cache));
	if (l1d == NULL) {
		cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches",
			usable_processors * sizeof(struct cpuinfo_cache), usable_processors);
		goto cleanup;
	}

	uint32_t l2_count = cluster_count;
	l2 = calloc(l2_count, sizeof(struct cpuinfo_cache));
	if (l2 == NULL) {
		cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches",
			l2_count * sizeof(struct cpuinfo_cache), l2_count);
		goto cleanup;
	}

	/* Populate cache infromation structures in l1i, l1d, and l2 */
	struct cpuinfo_cache shared_l2;	
	uint32_t cluster_id = UINT32_MAX;
	for (uint32_t i = 0; i < usable_processors; i++) {
		if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
			cluster_id++;
		}

		processors[i].smt_id = 0;
		processors[i].core = cores + i;
		processors[i].package = &package;
		processors[i].linux_id = (int) arm_linux_processors[i].system_processor_id;
		processors[i].cache.l1i = l1i + i;
		processors[i].cache.l1d = l1d + i;
		processors[i].cache.l2 = l2 + cluster_id;
		linux_cpu_to_processor_map[arm_linux_processors[i].system_processor_id] = &processors[i];

		cores[i].processor_start = i;
		cores[i].processor_count = 1;
		cores[i].core_id = i;
		cores[i].package = &package;
		cores[i].vendor = arm_linux_processors[i].vendor;
		cores[i].uarch = arm_linux_processors[i].uarch;
		cores[i].midr = arm_linux_processors[i].midr;
		linux_cpu_to_core_map[arm_linux_processors[i].system_processor_id] = &cores[i];

		cpuinfo_arm_decode_cache(
			arm_linux_processors[i].uarch,
			arm_linux_processors[i].package_processor_count,
			arm_linux_processors[i].midr,
			&chipset,
			cluster_id,
			arm_linux_processors[i].architecture_version,
			&l1i[i], &l1d[i], &shared_l2);
		l1i[i].processor_start = l1d[i].processor_start = i;
		l1i[i].processor_count = l1d[i].processor_count = 1;
		#if CPUINFO_ARCH_ARM
			/* L1I reported in /proc/cpuinfo overrides defaults */
			if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ICACHE)) {
				l1i[i] = (struct cpuinfo_cache) {
					.size = arm_linux_processors[i].proc_cpuinfo_cache.i_size,
					.associativity = arm_linux_processors[i].proc_cpuinfo_cache.i_assoc,
					.sets = arm_linux_processors[i].proc_cpuinfo_cache.i_sets,
					.partitions = 1,
					.line_size = arm_linux_processors[i].proc_cpuinfo_cache.i_line_length
				};
			}
			/* L1D reported in /proc/cpuinfo overrides defaults */
			if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_DCACHE)) {
				l1d[i] = (struct cpuinfo_cache) {
					.size = arm_linux_processors[i].proc_cpuinfo_cache.d_size,
					.associativity = arm_linux_processors[i].proc_cpuinfo_cache.d_assoc,
					.sets = arm_linux_processors[i].proc_cpuinfo_cache.d_sets,
					.partitions = 1,
					.line_size = arm_linux_processors[i].proc_cpuinfo_cache.d_line_length
				};
			}
		#endif
		if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
			shared_l2.processor_start = i;
			shared_l2.processor_count = arm_linux_processors[i].package_processor_count;
			l2[cluster_id] = shared_l2;
		}
	}

	if (cluster_count == 1 && l2[0].size == 0) {
		/* CPU without L2 cache */
		free(l2);
		l2 = NULL;
		l2_count = 0;
	}

	/* Commit */
	cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map;
	cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map;
	cpuinfo_processors = processors;
	cpuinfo_cores = cores;
	cpuinfo_packages = &package;
	cpuinfo_cache[cpuinfo_cache_level_1i] = l1i;
	cpuinfo_cache[cpuinfo_cache_level_1d] = l1d;
	cpuinfo_cache[cpuinfo_cache_level_2]  = l2;

	cpuinfo_processors_count = cpuinfo_cores_count = usable_processors;
	cpuinfo_packages_count = 1;
	cpuinfo_cache_count[cpuinfo_cache_level_1i] = usable_processors;
	cpuinfo_cache_count[cpuinfo_cache_level_1d] = usable_processors;
	cpuinfo_cache_count[cpuinfo_cache_level_2]  = l2_count;

	linux_cpu_to_processor_map = NULL;
	linux_cpu_to_core_map = NULL;
	processors = NULL;
	cores = NULL;
	l1i = l1d = l2 = NULL;

	#ifdef __ANDROID__
		cpuinfo_gpu_query_gles2(package.gpu_name);
	#endif

cleanup:
	free(arm_linux_processors);
	free(linux_cpu_to_processor_map);
	free(linux_cpu_to_core_map);
	free(processors);
	free(cores);
	free(l1i);
	free(l1d);
	free(l2);
}