Reorganize source tree: kernel/, v3/, v4/ split and board infrastructure
Source tree reorganization: - Move StarForth v3 engine to v3/ (src/, include/, Makefile) - Move kernel to kernel/ (src/, include/, linker/, Makefile) - Create v4/ skeleton for F18-ISA golden model (DECOMPOSITION.md, JUSTIFICATION.md) - Move FABRIC-0..4.md to docs/fabric/ - Move ONTOLOGY.md and ROADMAP.md to docs/ Board infrastructure: - Add boards/ser5/, boards/raspi/, boards/milkv/, boards/zynq7020/ - Each board has board.mk (ISA, CPU flags, boot recipe) and README.md - Root Makefile becomes thin dispatcher: boot_image, all, clean, docs take TARGET - make boot_image TARGET=SER5|RASPI|MILKV builds one GPT/MBR image per board - ZYNQ7020 target exists but stops with clear error (ARMv7 port not built yet) - scripts/mkdiskimage.sh builds disk images for all boards Docs pipeline: - docs/book/ with LaTeX master (main.tex) and Makefile - pandoc converts Markdown to LaTeX at build time - Two Lua filters: table-widths.lua (wide tables wrap), code-breaks.lua (inline code breaks) - make docs builds single PDF (754 pages, 0 missing characters) - make docs TARGET=<board> adds board appendix - build/docs/<book|board>/meta.tex stamps git commit into PDF Bug fixes: - 42 include paths that only worked by accident now use correct relative paths - clang-18 hardcode replaced with configurable CC variable (fixed aarch64 build) - Pi 5: kernel_2712.img linked at 0x80000, .bss zeroed, memory reserved - Doxyfile, .clang-tidy, README.md, Kconfig paths updated Verified: - Hosted v3 build passes 1012 tests, 0 failures - SER5 image boots in QEMU (OVMF), POST passes, K exact (65536 = Q48_ONE) - Milk-V image boots in QEMU (OpenSBI + U-Boot + bootefi), POST passes - make clean TARGET=<board> removes only that board and its ISA objects - make all builds all boards, hosted v3, and docs in one run Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
rajames
@@ -0,0 +1,783 @@
|
||||
/*
|
||||
StarForth — Steady-State Virtual Machine Runtime
|
||||
|
||||
Copyright (c) 2023–2025 Robert A. James
|
||||
All rights reserved.
|
||||
|
||||
This file is part of the StarForth project.
|
||||
|
||||
Licensed under the StarForth License, Version 1.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
You may obtain a copy of the License at:
|
||||
https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
|
||||
|
||||
This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
express or implied, including but not limited to the warranties of
|
||||
merchantability, fitness for a particular purpose, and noninfringement.
|
||||
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
StarForth — Steady-State Virtual Machine Runtime
|
||||
Copyright (c) 2023–2025 Robert A. James
|
||||
All rights reserved.
|
||||
|
||||
This file is part of the StarForth project.
|
||||
|
||||
Licensed under the StarForth License, Version 1.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
You may obtain a copy of the License at:
|
||||
https://github.com/star.4th@proton.me/StarForth/LICENSE.txt
|
||||
|
||||
This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
express or implied, including but not limited to the warranties of
|
||||
merchantability, fitness for a particular purpose, and noninfringement.
|
||||
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
#ifndef VM_ASM_OPT_ARM64_H
|
||||
#define VM_ASM_OPT_ARM64_H
|
||||
|
||||
#include "vm.h"
|
||||
#include <stdint.h>
|
||||
#ifdef __linux__
|
||||
#include <sys/auxv.h> /* For getauxval, AT_HWCAP */
|
||||
#include <asm/hwcap.h> /* For HWCAP_ASIMD */
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Configuration: Define USE_ASM_OPT=1 to enable assembly optimizations
|
||||
*/
|
||||
#ifndef USE_ASM_OPT
|
||||
#define USE_ASM_OPT 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
* ARM64 Architecture Notes:
|
||||
* =========================
|
||||
*
|
||||
* Registers (64-bit):
|
||||
* - x0-x7: Argument/result registers
|
||||
* - x8: Indirect result location
|
||||
* - x9-x15: Temporary registers (caller-saved)
|
||||
* - x16-x17: Intra-procedure-call temporary (IP0, IP1)
|
||||
* - x18: Platform register (reserved on some platforms)
|
||||
* - x19-x28: Callee-saved registers (must preserve)
|
||||
* - x29: Frame pointer (FP)
|
||||
* - x30: Link register (LR)
|
||||
* - sp: Stack pointer
|
||||
*
|
||||
* 32-bit variants: w0-w30 (lower 32 bits of x0-x30)
|
||||
*
|
||||
* Advantages over x86_64:
|
||||
* - More registers (31 general-purpose vs 16)
|
||||
* - Conditional execution on most instructions
|
||||
* - Load/store with auto-increment
|
||||
* - Better power efficiency
|
||||
*
|
||||
* Raspberry Pi 4 specifics:
|
||||
* - Cortex-A72 CPU (4 cores @ 1.5GHz)
|
||||
* - ARMv8-A architecture
|
||||
* - 32KB L1I + 32KB L1D per core
|
||||
* - 1MB shared L2 cache
|
||||
* - NEON SIMD support
|
||||
*/
|
||||
|
||||
/* ============================================================================
|
||||
* OPTIMIZATION 1: Stack Operations (vm_push, vm_pop, vm_rpush, vm_rpop)
|
||||
* ============================================================================
|
||||
*
|
||||
* ARM64 advantages:
|
||||
* - Conditional execution eliminates branches
|
||||
* - Load/store with immediate offset
|
||||
* - Post-increment addressing modes
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
#if USE_ASM_OPT
|
||||
|
||||
/* vm_push optimized - Data stack push */
|
||||
/**
|
||||
* @brief Optimized assembly implementation of data stack push operation
|
||||
* @param vm Pointer to VM instance
|
||||
* @param value Value to push onto data stack
|
||||
* @note Sets vm->error on stack overflow
|
||||
*/
|
||||
static inline void vm_push_asm(VM *vm, cell_t value) {
|
||||
int error = 0;
|
||||
|
||||
__asm__ __volatile__(
|
||||
/* Load current dsp */
|
||||
"ldr w0, %[dsp]\n\t"
|
||||
|
||||
/* Check for overflow: dsp >= 1022 */
|
||||
"cmp w0, #1022\n\t"
|
||||
"b.ge 1f\n\t" /* Branch if >= */
|
||||
|
||||
/* No overflow: increment dsp */
|
||||
"add w1, w0, #1\n\t" /* w1 = dsp + 1 */
|
||||
"str w1, %[dsp]\n\t"
|
||||
|
||||
/* Store value: vm->data_stack[dsp+1] = value */
|
||||
/* Calculate address: base + (dsp+1)*8 */
|
||||
"str %[val], [%[stack], w1, sxtw #3]\n\t" /* base + index*8 */
|
||||
|
||||
/* Set error=0 in success path */
|
||||
"mov w3, #0\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
"b 2f\n\t"
|
||||
|
||||
/* Overflow path */
|
||||
"1:\n\t"
|
||||
"mov w3, #1\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
|
||||
"2:\n\t"
|
||||
: [dsp]"+m"(vm->dsp),
|
||||
[err]"=m"(error)
|
||||
: [val]"r"(value),
|
||||
[stack]"r"(vm->data_stack)
|
||||
: "x0", "x1", "x3", "cc", "memory"
|
||||
);
|
||||
|
||||
if (__builtin_expect(error, 0)) {
|
||||
vm->error = 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* vm_pop optimized - Data stack pop */
|
||||
/**
|
||||
* @brief Optimized assembly implementation of data stack pop operation
|
||||
* @param vm Pointer to VM instance
|
||||
* @return Popped value, or 0 on underflow
|
||||
* @note Sets vm->error on stack underflow
|
||||
*/
|
||||
static inline cell_t vm_pop_asm(VM *vm) {
|
||||
cell_t value = 0;
|
||||
int error = 0;
|
||||
|
||||
__asm__ __volatile__(
|
||||
/* Load current dsp */
|
||||
"ldr w0, %[dsp]\n\t"
|
||||
|
||||
/* Check for underflow: dsp < 0 */
|
||||
"cmp w0, #0\n\t"
|
||||
"b.lt 1f\n\t" /* Branch if less than */
|
||||
|
||||
/* No underflow: load value */
|
||||
"ldr %[val], [%[stack], w0, sxtw #3]\n\t" /* base + index*8 */
|
||||
|
||||
/* Decrement dsp */
|
||||
"sub w1, w0, #1\n\t"
|
||||
"str w1, %[dsp]\n\t"
|
||||
|
||||
/* Set error=0 in success path */
|
||||
"mov w3, #0\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
"b 2f\n\t"
|
||||
|
||||
/* Underflow path */
|
||||
"1:\n\t"
|
||||
"mov w3, #1\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
"mov %[val], #0\n\t" /* Return 0 on error */
|
||||
|
||||
"2:\n\t"
|
||||
: [dsp]"+m"(vm->dsp),
|
||||
[val]"=r"(value),
|
||||
[err]"=m"(error)
|
||||
: [stack]"r"(vm->data_stack)
|
||||
: "x0", "x1", "x3", "cc", "memory"
|
||||
);
|
||||
|
||||
if (__builtin_expect(error, 0)) {
|
||||
vm->error = 1;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/* vm_rpush optimized - Return stack push */
|
||||
/**
|
||||
* @brief Optimized assembly implementation of return stack push operation
|
||||
* @param vm Pointer to VM instance
|
||||
* @param value Value to push onto return stack
|
||||
* @note Sets vm->error on stack overflow
|
||||
*/
|
||||
static inline void vm_rpush_asm(VM *vm, cell_t value) {
|
||||
int error = 0;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"ldr w0, %[rsp]\n\t"
|
||||
"cmp w0, #1022\n\t"
|
||||
"b.ge 1f\n\t"
|
||||
"add w1, w0, #1\n\t"
|
||||
"str w1, %[rsp]\n\t"
|
||||
"str %[val], [%[stack], w1, sxtw #3]\n\t"
|
||||
"mov w3, #0\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
"b 2f\n\t"
|
||||
"1:\n\t"
|
||||
"mov w3, #1\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
"2:\n\t"
|
||||
: [rsp]"+m"(vm->rsp),
|
||||
[err]"=m"(error)
|
||||
: [val]"r"(value),
|
||||
[stack]"r"(vm->return_stack)
|
||||
: "x0", "x1", "x3", "cc", "memory"
|
||||
);
|
||||
|
||||
if (__builtin_expect(error, 0)) {
|
||||
vm->error = 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* vm_rpop optimized - Return stack pop */
|
||||
/**
|
||||
* @brief Optimized assembly implementation of return stack pop operation
|
||||
* @param vm Pointer to VM instance
|
||||
* @return Popped value, or 0 on underflow
|
||||
* @note Sets vm->error on stack underflow
|
||||
*/
|
||||
static inline cell_t vm_rpop_asm(VM *vm) {
|
||||
cell_t value = 0;
|
||||
int error = 0;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"ldr w0, %[rsp]\n\t"
|
||||
"cmp w0, #0\n\t"
|
||||
"b.lt 1f\n\t"
|
||||
"ldr %[val], [%[stack], w0, sxtw #3]\n\t"
|
||||
"sub w1, w0, #1\n\t"
|
||||
"str w1, %[rsp]\n\t"
|
||||
"mov w3, #0\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
"b 2f\n\t"
|
||||
"1:\n\t"
|
||||
"mov w3, #1\n\t"
|
||||
"str w3, %[err]\n\t"
|
||||
"mov %[val], #0\n\t"
|
||||
"2:\n\t"
|
||||
: [rsp]"+m"(vm->rsp),
|
||||
[val]"=r"(value),
|
||||
[err]"=m"(error)
|
||||
: [stack]"r"(vm->return_stack)
|
||||
: "x0", "x1", "x3", "cc", "memory"
|
||||
);
|
||||
|
||||
if (__builtin_expect(error, 0)) {
|
||||
vm->error = 1;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
* OPTIMIZATION 2: Fast Arithmetic with Overflow Detection
|
||||
* ============================================================================
|
||||
*
|
||||
* ARM64 has excellent support for overflow detection via condition flags
|
||||
* and conditional select instructions
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
/* Fast add with overflow detection using ADDS and conditional select */
|
||||
/**
|
||||
* @brief Fast addition with overflow detection using ARM64 ADDS instruction
|
||||
* @param a First operand
|
||||
* @param b Second operand
|
||||
* @param result Pointer to store result
|
||||
* @return 1 if overflow occurred, 0 otherwise
|
||||
*/
|
||||
static inline int vm_add_check_overflow(cell_t a, cell_t b, cell_t *result) {
|
||||
cell_t res;
|
||||
int overflow;
|
||||
|
||||
__asm__(
|
||||
"adds %[res], %[a], %[b]\n\t" /* Add and set flags */
|
||||
"cset %w[ovf], vs\n\t" /* Set overflow flag (V flag) */
|
||||
: [res]"=r"(res),
|
||||
[ovf]"=r"(overflow)
|
||||
: [a]"r"(a),
|
||||
[b]"r"(b)
|
||||
: "cc"
|
||||
);
|
||||
|
||||
*result = res;
|
||||
return overflow;
|
||||
}
|
||||
|
||||
/* Fast subtract with overflow detection */
|
||||
/**
|
||||
* @brief Fast subtraction with overflow detection using ARM64 SUBS instruction
|
||||
* @param a First operand
|
||||
* @param b Second operand
|
||||
* @param result Pointer to store result
|
||||
* @return 1 if overflow occurred, 0 otherwise
|
||||
*/
|
||||
static inline int vm_sub_check_overflow(cell_t a, cell_t b, cell_t *result) {
|
||||
cell_t res;
|
||||
int overflow;
|
||||
|
||||
__asm__(
|
||||
"subs %[res], %[a], %[b]\n\t"
|
||||
"cset %w[ovf], vs\n\t"
|
||||
: [res]"=r"(res),
|
||||
[ovf]"=r"(overflow)
|
||||
: [a]"r"(a),
|
||||
[b]"r"(b)
|
||||
: "cc"
|
||||
);
|
||||
|
||||
*result = res;
|
||||
return overflow;
|
||||
}
|
||||
|
||||
/* Fast multiply with high-word result (for star-slash-MOD) */
|
||||
/**
|
||||
* @brief Fast 128-bit multiplication using ARM64 SMULH instruction
|
||||
* @param a First operand
|
||||
* @param b Second operand
|
||||
* @param hi Pointer to store high 64 bits of result
|
||||
* @param lo Pointer to store low 64 bits of result
|
||||
*/
|
||||
static inline void vm_mul_double(cell_t a, cell_t b, cell_t *hi, cell_t *lo) {
|
||||
__asm__(
|
||||
"mul %[lo], %[a], %[b]\n\t" /* Low 64 bits */
|
||||
"smulh %[hi], %[a], %[b]\n\t" /* High 64 bits (signed) */
|
||||
: [lo]"=r"(*lo),
|
||||
[hi]"=r"(*hi)
|
||||
: [a]"r"(a),
|
||||
[b]"r"(b)
|
||||
);
|
||||
}
|
||||
|
||||
/* Fast divide with remainder (for /MOD) */
|
||||
/**
|
||||
* @brief Fast division with remainder using ARM64 SDIV and MSUB instructions
|
||||
* @param dividend Dividend value
|
||||
* @param divisor Divisor value
|
||||
* @param quotient Pointer to store quotient
|
||||
* @param remainder Pointer to store remainder
|
||||
*/
|
||||
static inline void vm_divmod(cell_t dividend, cell_t divisor,
|
||||
cell_t *quotient, cell_t *remainder) {
|
||||
cell_t quot, rem;
|
||||
|
||||
__asm__(
|
||||
"sdiv %[quot], %[dividend], %[divisor]\n\t" /* Quotient */
|
||||
"msub %[rem], %[quot], %[divisor], %[dividend]\n\t" /* Remainder = dividend - quot*divisor */
|
||||
: [quot]"=r"(quot),
|
||||
[rem]"=r"(rem)
|
||||
: [dividend]"r"(dividend),
|
||||
[divisor]"r"(divisor)
|
||||
);
|
||||
|
||||
*quotient = quot;
|
||||
*remainder = rem;
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
* OPTIMIZATION 3: Fast String Operations for Dictionary Lookup
|
||||
* ============================================================================
|
||||
*
|
||||
* ARM64 doesn't have rep-style instructions, but we can:
|
||||
* - Use NEON for parallel comparison
|
||||
* - Use load-multiple for cache efficiency
|
||||
* - Unroll loops for small strings
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
/* Fast string comparison for short strings (<=32 bytes) */
|
||||
/**
|
||||
* @brief Optimized string comparison for short strings (<=32 bytes)
|
||||
* @param s1 First string
|
||||
* @param s2 Second string
|
||||
* @param len Length to compare
|
||||
* @return 0 if equal, 1 if different
|
||||
*/
|
||||
static inline int vm_strcmp_short_asm(const char *s1, const char *s2, size_t len) {
|
||||
int result = 0;
|
||||
|
||||
if (len <= 8) {
|
||||
/* Use single 64-bit load for strings <= 8 bytes */
|
||||
__asm__(
|
||||
"mov x0, #0\n\t" /* result = 0 */
|
||||
"cbz %[len], 2f\n\t" /* if len==0, equal */
|
||||
"ldr x1, [%[s1]]\n\t"
|
||||
"ldr x2, [%[s2]]\n\t"
|
||||
"eor x3, x1, x2\n\t" /* XOR to find differences */
|
||||
|
||||
/* Mask off bits beyond length */
|
||||
"mov x4, #64\n\t"
|
||||
"lsl x5, %[len], #3\n\t" /* len * 8 */
|
||||
"sub x4, x4, x5\n\t"
|
||||
"lsr x1, x1, x4\n\t"
|
||||
"lsr x2, x2, x4\n\t"
|
||||
"eor x3, x1, x2\n\t"
|
||||
|
||||
"cmp x3, #0\n\t"
|
||||
"cset %w[res], ne\n\t" /* result = (x3 != 0) */
|
||||
"2:\n\t"
|
||||
: [res]"=r"(result)
|
||||
: [s1]"r"(s1),
|
||||
[s2]"r"(s2),
|
||||
[len]"r"(len)
|
||||
: "x0", "x1", "x2", "x3", "x4", "x5", "cc", "memory"
|
||||
);
|
||||
} else {
|
||||
/* Fallback to byte-by-byte for longer strings */
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
if (s1[i] != s2[i]) {
|
||||
result = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Fast string comparison using NEON (for longer strings) */
|
||||
#ifdef __ARM_NEON
|
||||
static inline int vm_strcmp_neon(const char *s1, const char *s2, size_t len) {
|
||||
int result = 0;
|
||||
|
||||
/* Process 16 bytes at a time using NEON */
|
||||
while (len >= 16) {
|
||||
__asm__ __volatile__(
|
||||
"ld1 {v0.16b}, [%[s1]], #16\n\t" /* Load 16 bytes from s1, post-increment */
|
||||
"ld1 {v1.16b}, [%[s2]], #16\n\t" /* Load 16 bytes from s2, post-increment */
|
||||
"cmeq v2.16b, v0.16b, v1.16b\n\t" /* Compare equal */
|
||||
"uminv b3, v2.16b\n\t" /* Get minimum (all must be 0xFF for equal) */
|
||||
"fmov %w[res], s3\n\t" /* Move to general register */
|
||||
"cmp %w[res], #0xff\n\t"
|
||||
"b.ne 1f\n\t" /* If not all equal, exit */
|
||||
"1:\n\t"
|
||||
: [res]"=r"(result),
|
||||
[s1]"+r"(s1),
|
||||
[s2]"+r"(s2)
|
||||
:
|
||||
: "v0", "v1", "v2", "v3", "cc", "memory"
|
||||
);
|
||||
|
||||
if (result != 0xFF) {
|
||||
return 1; /* Not equal */
|
||||
}
|
||||
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
/* Handle remaining bytes */
|
||||
while (len > 0) {
|
||||
if (*s1++ != *s2++) return 1;
|
||||
len--;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Main string comparison wrapper */
|
||||
static inline int vm_strcmp_asm(const char *s1, const char *s2, size_t len) {
|
||||
|
||||
|
||||
|
||||
#ifdef __ARM_NEON
|
||||
if (len>= 16) {
|
||||
return vm_strcmp_neon(s1, s2, len);
|
||||
}
|
||||
#endif
|
||||
return vm_strcmp_short_asm(s1, s2, len);
|
||||
}
|
||||
|
||||
/* Fast memory copy optimized for ARM64 */
|
||||
/**
|
||||
* @brief Optimized memory copy using ARM64 load/store pair instructions
|
||||
* @param dest Destination buffer
|
||||
* @param src Source buffer
|
||||
* @param len Number of bytes to copy
|
||||
*/
|
||||
static inline void vm_memcpy_asm(void *dest, const void *src, size_t len) {
|
||||
/* ARM64 has efficient load/store pair instructions */
|
||||
if (len >= 16) {
|
||||
__asm__ __volatile__(
|
||||
"1:\n\t"
|
||||
"ldp x0, x1, [%[src]], #16\n\t" /* Load pair, post-increment */
|
||||
"stp x0, x1, [%[dst]], #16\n\t" /* Store pair, post-increment */
|
||||
"subs %[len], %[len], #16\n\t"
|
||||
"b.ge 1b\n\t"
|
||||
: [src]"+r"(src),
|
||||
[dst]"+r"(dest),
|
||||
[len]"+r"(len)
|
||||
:
|
||||
: "x0", "x1", "cc", "memory"
|
||||
);
|
||||
len += 16; /* Adjust for overshoot */
|
||||
}
|
||||
|
||||
/* Copy remaining bytes */
|
||||
char *d = (char *) dest;
|
||||
const char *s = (const char *) src;
|
||||
while (len-- > 0) {
|
||||
*d++ = *s++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Fast memory zero using store pair */
|
||||
/**
|
||||
* @brief Fast memory zeroing using ARM64 store pair instructions
|
||||
* @param dest Buffer to zero
|
||||
* @param len Number of bytes to zero
|
||||
*/
|
||||
static inline void vm_memzero_asm(void *dest, size_t len) {
|
||||
if (len >= 16) {
|
||||
__asm__ __volatile__(
|
||||
"1:\n\t"
|
||||
"stp xzr, xzr, [%[dst]], #16\n\t" /* Store zero pair */
|
||||
"subs %[len], %[len], #16\n\t"
|
||||
"b.ge 1b\n\t"
|
||||
: [dst]"+r"(dest),
|
||||
[len]"+r"(len)
|
||||
:
|
||||
: "cc", "memory"
|
||||
);
|
||||
len += 16;
|
||||
}
|
||||
|
||||
/* Zero remaining bytes */
|
||||
char *d = (char *) dest;
|
||||
while (len-- > 0) {
|
||||
*d++ = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
* OPTIMIZATION 4: Branchless Min/Max using CSEL
|
||||
* ============================================================================
|
||||
*
|
||||
* ARM64's conditional select is perfect for branchless operations
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
static inline cell_t vm_min_asm(cell_t a, cell_t b) {
|
||||
cell_t result;
|
||||
__asm__(
|
||||
"cmp %[a], %[b]\n\t"
|
||||
"csel %[res], %[a], %[b], lt\n\t" /* Select a if a < b, else b */
|
||||
: [res]"=r"(result)
|
||||
: [a]"r"(a),
|
||||
[b]"r"(b)
|
||||
: "cc"
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
static inline cell_t vm_max_asm(cell_t a, cell_t b) {
|
||||
cell_t result;
|
||||
__asm__(
|
||||
"cmp %[a], %[b]\n\t"
|
||||
"csel %[res], %[a], %[b], gt\n\t" /* Select a if a > b, else b */
|
||||
: [res]"=r"(result)
|
||||
: [a]"r"(a),
|
||||
[b]"r"(b)
|
||||
: "cc"
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Absolute value using conditional negate */
|
||||
static inline cell_t vm_abs_asm(cell_t a) {
|
||||
cell_t result;
|
||||
__asm__(
|
||||
"cmp %[a], #0\n\t"
|
||||
"cneg %[res], %[a], lt\n\t" /* Negate if negative */
|
||||
: [res]"=r"(result)
|
||||
: [a]"r"(a)
|
||||
: "cc"
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
* OPTIMIZATION 5: Bit Manipulation
|
||||
* ============================================================================
|
||||
*
|
||||
* ARM64 has excellent bit manipulation instructions
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
/* Count leading zeros */
|
||||
static inline int vm_clz(cell_t x) {
|
||||
int result;
|
||||
__asm__("clz %[res], %[x]": [res]"=r"(result): [x]"r"(x));
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Count trailing zeros */
|
||||
static inline int vm_ctz(cell_t x) {
|
||||
int result;
|
||||
__asm__(
|
||||
"rbit %[tmp], %[x]\n\t" /* Reverse bits */
|
||||
"clz %[res], %[tmp]\n\t" /* Count leading zeros of reversed */
|
||||
: [res]"=r"(result),
|
||||
[tmp]"=r"(x)
|
||||
: [x]"r"(x)
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Population count (count set bits) */
|
||||
static inline int vm_popcnt(cell_t x) {
|
||||
int result;
|
||||
__asm__(
|
||||
"fmov d0, %[x]\n\t" /* Move to NEON register */
|
||||
"cnt v0.8b, v0.8b\n\t" /* Count bits in each byte */
|
||||
"addv b0, v0.8b\n\t" /* Sum all bytes */
|
||||
"fmov %w[res], s0\n\t" /* Move back to general register */
|
||||
: [res]"=r"(result)
|
||||
: [x]"r"(x)
|
||||
: "v0"
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
* OPTIMIZATION 6: CPU Feature Detection
|
||||
* ============================================================================
|
||||
*
|
||||
* ARM64 uses a different mechanism than x86 CPUID
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
/* Read ARM64 CPU ID register */
|
||||
static inline uint64_t vm_read_midr(void) {
|
||||
uint64_t midr;
|
||||
__asm__("mrs %0, midr_el1": "=r"(midr));
|
||||
return midr;
|
||||
}
|
||||
|
||||
/* Check for specific CPU features via auxiliary control register */
|
||||
static inline int vm_has_neon(void) {
|
||||
|
||||
|
||||
|
||||
#ifdef __ARM_NEON
|
||||
return 1;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Check for Cortex-A72 (Raspberry Pi 4) */
|
||||
static inline int vm_is_cortex_a72(void) {
|
||||
|
||||
|
||||
|
||||
/* Cortex-A72 MIDR: 0x410FD083 */
|
||||
/* Note: Reading MIDR requires EL1, may not work in userspace */
|
||||
/* Use hwcaps instead for userspace detection */
|
||||
#ifdef __linux__
|
||||
unsigned long hwcap = getauxval(AT_HWCAP);
|
||||
return (hwcap &HWCAP_ASIMD)!= 0; /* ASIMD = Advanced SIMD (NEON) */
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* ============================================================================
|
||||
* OPTIMIZATION 7: Cache Management (Raspberry Pi 4 specific)
|
||||
* ============================================================================
|
||||
*
|
||||
* Raspberry Pi 4 cache hierarchy:
|
||||
* - L1I: 32KB (3-way, 64-byte lines)
|
||||
* - L1D: 32KB (2-way, 64-byte lines)
|
||||
* - L2: 1MB (16-way, 64-byte lines)
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
/* Prefetch data into cache */
|
||||
static inline void vm_prefetch(const void *addr) {
|
||||
__asm__ __volatile__(
|
||||
"prfm pldl1keep, [%0]\n\t" /* Prefetch for load, L1 cache, temporal */
|
||||
:
|
||||
: "r"(addr)
|
||||
: "memory"
|
||||
);
|
||||
}
|
||||
|
||||
/* Prefetch with streaming hint (for sequential access) */
|
||||
static inline void vm_prefetch_stream(const void *addr) {
|
||||
__asm__ __volatile__(
|
||||
"prfm pldl1strm, [%0]\n\t" /* Prefetch for load, L1 cache, streaming */
|
||||
:
|
||||
: "r"(addr)
|
||||
: "memory"
|
||||
);
|
||||
}
|
||||
|
||||
/* Data cache zero (fast way to zero cache lines) */
|
||||
static inline void vm_dc_zva(void *addr) {
|
||||
__asm__ __volatile__(
|
||||
"dc zva, %0\n\t" /* Zero entire cache line (64 bytes) */
|
||||
:
|
||||
: "r"(addr)
|
||||
: "memory"
|
||||
);
|
||||
}
|
||||
|
||||
#else /* !USE_ASM_OPT */
|
||||
|
||||
/* Fallback to standard implementations */
|
||||
#define vm_push_asm(vm, val) vm_push(vm, val)
|
||||
#define vm_pop_asm(vm) vm_pop(vm)
|
||||
#define vm_rpush_asm(vm, val) vm_rpush(vm, val)
|
||||
#define vm_rpop_asm(vm) vm_rpop(vm)
|
||||
|
||||
#endif /* USE_ASM_OPT */
|
||||
|
||||
/* ============================================================================
|
||||
* RASPBERRY PI 4 SPECIFIC NOTES
|
||||
* ============================================================================
|
||||
*
|
||||
* CPU: Broadcom BCM2711 (Quad-core Cortex-A72 @ 1.5GHz)
|
||||
* Architecture: ARMv8-A
|
||||
* L1 Cache: 32KB I + 32KB D per core
|
||||
* L2 Cache: 1MB shared
|
||||
* RAM: 1GB/2GB/4GB/8GB LPDDR4-3200
|
||||
*
|
||||
* Optimization Tips:
|
||||
* 1. Align hot data to 64-byte cache lines
|
||||
* 2. Use NEON for parallel operations
|
||||
* 3. Prefetch sequential data access
|
||||
* 4. Keep hot code < 32KB (fits in L1I)
|
||||
* 5. Use load/store pairs (ldp/stp) for efficiency
|
||||
*
|
||||
* Thermal Throttling:
|
||||
* - CPU throttles at 80°C
|
||||
* - Add heatsink for sustained performance
|
||||
* - Monitor: vcgencmd measure_temp
|
||||
*
|
||||
* Build Flags:
|
||||
* -march=armv8-a+crc+simd -mtune=cortex-a72 -O3 -DUSE_ASM_OPT=1
|
||||
*
|
||||
* Cross-compilation:
|
||||
* CC=aarch64-linux-gnu-gcc
|
||||
*
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
#endif /* VM_ASM_OPT_ARM64_H */
|
||||
Reference in New Issue
Block a user