Simplify and refactor the bug215510 code.

This code is investigating the issue:

   https://bugzilla.kernel.org/show_bug.cgi?id=216610

This present commit extends x86_64 (aka amd64) support to 32-bit
arm build support. It is now possible cross compile the program
for the Raspberry Pi. To do this, the code needs 'docker' to work.

Signed-off-by: Andrew G. Morgan <morgan@kernel.org>

14 files changed, 309 insertions, 51 deletions

diff --git a/contrib/bug216610/.gitignore b/contrib/bug216610/.gitignore
new file mode 100644
index 0000000..1478d58
--- /dev/null
+++ b/contrib/bug216610/.gitignore
@@ -0,0 +1,3 @@
+*~
+arms
+Dockerfile
diff --git a/contrib/bug216610/Dockerfile b/contrib/bug216610/Dockerfile
new file mode 100644
index 0000000..5502b71
--- /dev/null
+++ b/contrib/bug216610/Dockerfile
@@ -0,0 +1,13 @@
+FROM debian:latest
+
+# A directory to share files via.
+RUN mkdir /shared
+
+RUN apt-get update
+RUN apt-get install -y gcc-arm-linux-gnueabi binutils-arm-linux-gnueabi
+RUN apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu
+
+# create a builder user
+RUN echo "builder:x:1000:1000:,,,:/home/builder:/bin/bash" >> /etc/passwd
+RUN echo "builder:*:19289:0:99999:7:::" >> /etc/shadow
+RUN mkdir -p /home/builder && chown builder.bin /home/builder
diff --git a/contrib/bug216610/Makefile b/contrib/bug216610/Makefile
index aca7ab9..ce96fb3 100644
--- a/contrib/bug216610/Makefile
+++ b/contrib/bug216610/Makefile
@@ -5,19 +5,26 @@ GOTARGET=$(shell eval $$(go env) ; echo $${GOHOSTOS}_$${GOARCH})
 
 all: go/fib
 
-go/fib: go/main.go go/vendor/fibber/fib.go go/vendor/fibber/fibs_$(GOTARGET).s go/vendor/fibber/fib_$(GOTARGET).syso go/vendor/kernel.org/pub/linux/libs/security/libcap/psx
-	cd go && CGO_ENABLED=0 go build -o fib main.go
+go/fib: go/main.go go/fibber/fib.go go/fibber/linkage.go go/fibber/fibs_$(GOTARGET).s go/fibber/fib_$(GOTARGET).syso
+	cd go && CGO_ENABLED=0 go build
 
-go/vendor/kernel.org/pub/linux/libs/security/libcap/psx:
-	mkdir -p go/vendor/kernel.org/pub/linux/libs/security/libcap/
-	ln -s $(topdir)/psx $@
+# Build the host native version.
+go/fibber/fib_$(GOTARGET).syso go/fibber/linkage.go: c/fib.c ./c/gcc.sh ./package_fns.sh
+	GCC=gcc ./c/gcc.sh -O3 c/fib.c -c -o go/fibber/fib_$(GOTARGET).syso
+	./package_fns.sh fibber go/fibber/fib_$(GOTARGET).syso > go/fibber/linkage.go
 
-go/vendor/fibber/fib_$(GOTARGET).syso: c/fib.c ./gcc_$(GOTARGET).sh
-	./gcc_$(GOTARGET).sh -O3 c/fib.c -c -o go/vendor/fibber/fib_$(GOTARGET).syso
+Dockerfile: Makefile ./mkdocker.sh
+	./mkdocker.sh > $@
+
+# Use this build target (make arms) to extend support to include arm
+# and arm64 GOARCH values.
+arms: Dockerfile Makefile ./c/gcc.sh ./c/build.sh ./c/fib.c
+	docker run --rm -v $$PWD/c:/shared:z -h debian -u $$(id -u) -it expt shared/build.sh
+	mv c/*.syso go/fibber/
+	touch arms
 
 clean:
-	rm -f *~
+	rm -f *~ arms
 	rm -f c/*.o c/*~
 	rm -f go/fib go/*~
-	rm -f go/vendor/fibber/*.syso go/vendor/fibber/*~
-	rm -rf go/vendor/kernel.org
+	rm -f go/fibber/*.syso go/fibber/*~ go/fibber/linkage.go
diff --git a/contrib/bug216610/README.md b/contrib/bug216610/README.md
index c158161..4425715 100644
--- a/contrib/bug216610/README.md
+++ b/contrib/bug216610/README.md
@@ -7,12 +7,12 @@ to include some C code, but not `libc` etc. For a long time, I had
 assumed this was not possible, since using `cgo` *requires* `libc` and
 `libpthread` linkage.
 
-This embedded compilation need was referenced in a [bug
+This _embedded compilation_ need was referenced in a [bug
 filed](https://bugzilla.kernel.org/show_bug.cgi?id=216610) against the
 [`"psx"`](https://pkg.go.dev/kernel.org/pub/linux/libs/security/libcap/psx)
 package. The bug-filer was seeking an alternative to `CGO_ENABLED=1`
-compilation needing the `cgo` variant of `psx` build. However, the go
-`"runtime"` package will always
+compilation _requiring_ the `cgo` variant of `psx` build. However, the
+go `"runtime"` package will always
 [`panic()`](https://cs.opensource.google/go/go/+/refs/tags/go1.19.2:src/runtime/os_linux.go;l=717-720)
 if you try this because it needs `libpthread` and `[g]libc` to work.
 
@@ -25,10 +25,11 @@ Wiki](https://zchee.github.io/golang-wiki/GcToolchainTricks/).
 This present directory evolved from my attempt to understand and
 hopefully resolve what was going on as reported in that bug into an
 example of this _trick_. I was unable to resolve the problem as
-reported because of the aformentioned `panic` in the Go
+reported because of the aformentioned `panic()` in the Go
 runtime. However, I was able to demonstrate embedding C code in a Go
-binary without use of cgo. So, a Go-native version of `"psx"` is thus
-achievable. This is what the example in this present directory does.
+binary _without_ use of cgo. In such a binary, the Go-native version
+of `"psx"` is thus achievable. This is what the example in this
+present directory demonstrates.
 
 *Caveat Emptor*: this example is very fragile. The Go team only
 supports `cgo` linking against C. That being said, I'd certainly like
@@ -42,23 +43,20 @@ In this example we have:
 - Some C code for the functions `fib_init()` and `fib_next()` that
 combine to implement a _compute engine_ to determine [Fibonacci
 Numbers](https://en.wikipedia.org/wiki/Fibonacci_number). The source
-for this is in the sub directory `./c/fib.c`.
+for this is in the sub directory `c/fib.c`.
 
-- Some Go code, in the directory `./go/vendor/fibber` that uses this
-C compiled compute kernel.
+- Some Go code, in the directory `go/fibber` that uses this C compiled
+compute kernel.
 
-- `gcc_linux_amd64.sh` which is a wrapper for `gcc` that adjusts the
-compilation to be digestible by Go's (internal) linker. Using `gcc`
-directly instead of this wrapper generates an incomplete binary -
-which miscomputes the expected answers. See the discussion below for
-what might be going on.
+- `c/gcc.sh` which is a wrapper for `gcc` that adjusts the compilation
+to be digestible by Go's (internal) linker (the one that gets invoked
+when compiling `CGO_ENABLED=0`. Using `gcc` directly instead of this
+wrapper generates an incomplete binary - which miscomputes the
+expected answers. See the discussion below for what seems to be going
+on.
 
 - A top level `Makefile` to build it all.
 
-This build uses vendored Go packages so one can experiment with
-modifications of the `"psx"` package to explore potential changes (of
-which there have been none).
-
 ## Building and running the built binary
 
 Set things up with:
@@ -85,34 +83,54 @@ The Fibonacci detail of what is going on is mostly uninteresting. The
 reason for developing this example was to explore the build issues in
 the reported [Bug
 216610](https://bugzilla.kernel.org/show_bug.cgi?id=216610). Ultimately,
-this example offers an alternative path to build a `nocgo` that links
-to compute engine style C code.
+this example offers an alternative path to building a `nocgo` program
+that links to compute kernel of C code.
 
-The reason we have added the `./gcc_linux_amd64.sh` wrapper for `gcc`
-is that we've found the Go linker has a hard time digesting the
+The reason we have added the `c/gcc.sh` wrapper for `gcc` is that
+we've found the Go linker has a hard time digesting the
 cross-sectional `%rip` based data addressing that various optimization
-modes of gcc like to use. Specifically, if a `R_X86_64_PC32`
-relocation entry made in a `.text` section is intended to map into a
-`.rodata.cst8` section in a generated `.syso` file, the Go linker
-seems to [replace this reference with a `0` offset to
+modes of gcc like to use. Specifically, in the x86_64/amd64
+architecture, if a `R_X86_64_PC32` relocation entry made in a `.text`
+section refers to an `.rodata.cst8` section in a generated `.syso`
+file, the Go linker seems to [replace this reference with a `0` offset
+to
 `(%rip)`](https://github.com/golang/go/issues/24321#issuecomment-1296084103). What
 our wrapper script does is rewrite the generated assembly to store
 these data references to the `.text` section. The Go linker has no
-problem with this _same section_ relative addressing.
+problem with this _same section_ relative addressing and is able to
+link the resulting objects without problems.
+
+If you want to cross compile, we have support for 32-bit arm
+compilation: what is needed for the Raspberry PI. To get this support,
+try:
+```
+$ make clean all arms
+$ cd go
+$ GOARCH=arm CGO_ENABLED=0 go build
+```
+The generated `fib` binary runs on a 32-bit Raspberry Pi.
 
 ## Future thoughts
 
-At present, this example only works on Linux with `x86_64` (in
-go-speak that is `linux_amd64`). This is because I have only provided
-some bridging assembly for Go to C calling conventions on that
-architecture target (`./go/vendor/fibber/fibs_linux_amd64.s`).
-
-Perhaps a later version will have bridging code for all the Go
-supported Linux architectures, but it will also have to provide some
-mechanism to build the `./c/fib.c` code to make
-`fib_linux_<arch>.syso` files. The [cited
-bug](https://bugzilla.kernel.org/show_bug.cgi?id=216610) includes some
-pointers for how to use Docker to support this.
+At present, this example only works on Linux with `x86_64` and `arm`
+build architectures. (In go-speak that is `linux_amd64` and
+`linux_arm`). This is because I have only provided some bridging
+assembly for Go to C calling conventions for those architecture
+targets: `./go/fibber/fibs_linux_amd64.s` and
+`./go/fibber/fibs_linux_arm.s`. The non-native, `make arms`, cross
+compilation requires the `docker` command to be available.
+
+I intend to implement an `arm64` build, when I have a system on which
+to test it.
+
+**Note** The Fedora system on which I've been developing this has some
+  SELINUX impediment to naively using the `docker -v ...` bind mount
+  option. I need the `:z` suffix for bind mounting. I don't know how
+  common an issue this is. On Fedora, building the arm variants of the
+  .syso file can be performed as follows:
+```
+$ docker run --rm -v $PWD/c:/shared:z -h debian -u $(id -u) -it expt shared/build.sh
+```
 
 ## Reporting bugs
 
diff --git a/contrib/bug216610/c/build.sh b/contrib/bug216610/c/build.sh
new file mode 100755
index 0000000..7458fb1
--- /dev/null
+++ b/contrib/bug216610/c/build.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+# Builds the following .syso files to the directory containing this script:
+#
+#   fib_linux_arm.syso
+#   fib_linux_arm64.syso
+
+cd ${0%/*}
+GCC=arm-linux-gnueabi-gcc ./gcc.sh -O3 fib.c -c -o fib_linux_arm.syso
+GCC=aarch64-linux-gnu-gcc ./gcc.sh -O3 fib.c -c -o fib_linux_arm64.syso
diff --git a/contrib/bug216610/c/gcc.sh b/contrib/bug216610/c/gcc.sh
new file mode 100755
index 0000000..33655d6
--- /dev/null
+++ b/contrib/bug216610/c/gcc.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# The Go linker does not seem to know what to do with relative
+# addressing of rodata.* offset from %rip. GCC likes to use this
+# addressing mode on this architecture, so we quickly run into
+# mis-computation when the relative addressing used in a .syso file of
+# symbol located data is resolved to completely the wrong place by the
+# Go (internal) linker.
+#
+# As a workaround for this, we can modify the assembly source code
+# generated by GCC to not point at problematic '.rodata.*' sections,
+# and place this data in the good old '.text' section where Go's
+# linker can make sense of it.
+#
+# This script exists to generate a '.syso' file from some '*.c' files.
+# It works by recognizing the '*.c' command line arguments and
+# converting them into fixed-up '*.s' files. It then performs the
+# compilation for the collection of the '*.s' files. Upon success, it
+# purges the intermediate '*.s' files.
+#
+# The fragile aspect of this present script is which compiler
+# arguments should be used for the compilation from '.c' -> '.s'
+# files. What we do is accumulate arguments until we encounter our
+# first '*.c' file and use those to perform the '.c' -> '.s'
+# compilation. We build up a complete command line for gcc
+# substituting '.s' files for '.c' files in the original command
+# line. Then with the new command line assembled we invoke gcc with
+# those. If that works, we remove all of the intermediate '.s' files.
+
+GCC="${GCC:=gcc}"
+setup=0
+args=()
+final=()
+ses=()
+
+for arg in "$@"; do
+    if [[ "${arg##*.}" = "c" ]]; then
+	setup=1
+	s="${arg%.*}.s"
+	"${GCC}" "${args[@]}" -S -o "${s}" "${arg}"
+	sed -i -e 's/.*\.rodata\..*/\t.text/' "${s}"
+	final+=("${s}")
+	ses+=("${s}")
+    else
+	if [[ $setup -eq 0 ]]; then
+	    args+=("${arg}")
+	fi
+	final+=("${arg}")
+    fi
+done
+
+#echo final: "${final[@]}"
+#echo args: "${args[@]}"
+#echo ses: "${ses[@]}"
+
+"${GCC}" "${final[@]}"
+if [[ $? -ne 0 ]]; then
+    echo "failed to compile"
+    exit 1
+fi
+rm -f "${ses[@]}"
diff --git a/contrib/bug216610/go/.gitignore b/contrib/bug216610/go/.gitignore
index 68b7ed0..ae14305 100644
--- a/contrib/bug216610/go/.gitignore
+++ b/contrib/bug216610/go/.gitignore
@@ -1,3 +1,5 @@
 fib
 *.syso
-vendor/kernel.org
+main
+go.sum
+linkage.go
diff --git a/contrib/bug216610/go/fibber/fib.go b/contrib/bug216610/go/fibber/fib.go
new file mode 100644
index 0000000..49757cd
--- /dev/null
+++ b/contrib/bug216610/go/fibber/fib.go
@@ -0,0 +1,32 @@
+// Package fibber implements a Fibonacci sequence generator using a C
+// coded compute kernel (a .syso file).
+package fibber
+
+import (
+	"unsafe"
+)
+
+// State is the native Go form of the C.state structure.
+type State struct {
+	B, A uint32
+}
+
+// cPtr converts State into a C pointer suitable as an argument for
+// sysoCaller.
+func (s *State) cPtr() unsafe.Pointer {
+	return unsafe.Pointer(&s.B)
+}
+
+// NewState initializes a Fibonacci Number sequence generator.  Upon
+// return s.A=0 and s.B=1 are the first two numbers in the sequence.
+func NewState() *State {
+	s := &State{}
+	syso__fib_init.call(s.cPtr())
+	return s
+}
+
+// Next advances the state to the next number in the sequence. Upon
+// return, s.B is the most recently calculated value.
+func (s *State) Next() {
+	syso__fib_next.call(s.cPtr())
+}
diff --git a/contrib/bug216610/go/fibber/fibs_linux_amd64.s b/contrib/bug216610/go/fibber/fibs_linux_amd64.s
new file mode 100644
index 0000000..5992d09
--- /dev/null
+++ b/contrib/bug216610/go/fibber/fibs_linux_amd64.s
@@ -0,0 +1,21 @@
+// To transition from a Go call to a C function call, we are skating
+// on really thin ice... Ceveat Emptor!
+//
+// Ref:
+//   https://gitlab.com/x86-psABIs/x86-64-ABI/-/wikis/home
+//
+// This is not strictly needed, but it makes gdb debugging less
+// confusing because spacer ends up being an alias for the TEXT
+// section start.
+TEXT ·spacer(SB),$0
+	RET
+
+#define RINDEX(n) (8*n)
+
+// Header to this function wrapper is the last time we can voluntarily
+// yield to some other goroutine.
+TEXT ·syso(SB),$0-16
+	MOVQ cFn+RINDEX(0)(FP), SI
+	MOVQ state+RINDEX(1)(FP), DI
+	CALL *SI
+	RET
diff --git a/contrib/bug216610/go/fibber/fibs_linux_arm.s b/contrib/bug216610/go/fibber/fibs_linux_arm.s
new file mode 100644
index 0000000..39640a5
--- /dev/null
+++ b/contrib/bug216610/go/fibber/fibs_linux_arm.s
@@ -0,0 +1,23 @@
+// To transition from a Go call to a C function call, we are skating
+// on really thin ice... Ceveat Emptor!
+//
+// Ref:
+//   https://stackoverflow.com/questions/261419/what-registers-to-save-in-the-arm-c-calling-convention
+//
+// This is not strictly needed, but it makes gdb debugging less
+// confusing because spacer ends up being an alias for the TEXT
+// section start.
+TEXT ·spacer(SB),$0
+	RET
+
+#define FINDEX(n) (8*n)
+
+// Header to this function wrapper is the last time we can voluntarily
+// yield to some other goroutine.
+//
+// Conventions: PC == R15, SP == R13, LR == R14, IP (scratch) = R12
+TEXT ·syso(SB),$0-8
+	MOVW	cFn+0(FP), R14
+	MOVW    state+4(FP), R0
+	BL	(R14)
+	RET
diff --git a/contrib/bug216610/go/go.mod b/contrib/bug216610/go/go.mod
index 819081e..28379e8 100644
--- a/contrib/bug216610/go/go.mod
+++ b/contrib/bug216610/go/go.mod
@@ -1,3 +1,5 @@
 module fib
 
 go 1.18
+
+require kernel.org/pub/linux/libs/security/libcap/psx v1.2.67
diff --git a/contrib/bug216610/go/main.go b/contrib/bug216610/go/main.go
index bb5a346..65121f6 100644
--- a/contrib/bug216610/go/main.go
+++ b/contrib/bug216610/go/main.go
@@ -3,11 +3,12 @@
 package main
 
 import (
-	"fibber"
 	"fmt"
 	"log"
 	"syscall"
 
+	"fib/fibber"
+
 	"kernel.org/pub/linux/libs/security/libcap/psx"
 )
 
@@ -20,7 +21,7 @@ func main() {
 	fmt.Println(pid)
 	s := fibber.NewState()
 	fmt.Print("fib: ", s.A, ", ", s.B)
-	for i:=0; i<8; i++ {
+	for i := 0; i < 8; i++ {
 		s.Next()
 		fmt.Print(", ", s.B)
 	}
diff --git a/contrib/bug216610/mkdocker.sh b/contrib/bug216610/mkdocker.sh
new file mode 100755
index 0000000..860c198
--- /dev/null
+++ b/contrib/bug216610/mkdocker.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# This script generates a Dockerfile to be used for cross-compilation
+cat <<EOF
+FROM debian:latest
+
+# A directory to share files via.
+RUN mkdir /shared
+
+RUN apt-get update
+RUN apt-get install -y gcc-arm-linux-gnueabi binutils-arm-linux-gnueabi
+RUN apt-get install -y gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu
+
+# create a builder user
+RUN echo "builder:x:$(id -u):$(id -g):,,,:/home/builder:/bin/bash" >> /etc/passwd
+RUN echo "builder:*:19289:0:99999:7:::" >> /etc/shadow
+RUN mkdir -p /home/builder && chown builder.bin /home/builder
+EOF
diff --git a/contrib/bug216610/package_fns.sh b/contrib/bug216610/package_fns.sh
new file mode 100755
index 0000000..0f4b91c
--- /dev/null
+++ b/contrib/bug216610/package_fns.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Generate some Go code to make calling into the C code of the .syso
+# file easier.
+
+package="${1}"
+syso="${2}"
+
+if [[ -z "${syso}" ]]; then
+    echo "usage: $0 <package> <.....syso>" >&2
+    exit 1
+fi
+
+if [[ "${syso%.syso}" == "${syso}" ]]; then
+    echo "2nd argument should be a .syso file" >&2
+    exit 1
+fi
+
+cat<<EOF
+package ${package}
+
+import (
+	"unsafe"
+)
+
+// syso is how we call, indirectly, into the C-code.
+func syso(cFn, state unsafe.Pointer)
+
+type sysoCaller struct {
+	ptr unsafe.Pointer
+}
+
+// call calls the syso linked C-function, $sym().
+func (s *sysoCaller) call(data unsafe.Pointer) {
+	syso(s.ptr, data)
+}
+EOF
+
+for sym in $(objdump -x "${syso}" | grep -F 'g     F' | awk '{print $6}'); do
+    cat<<EOF
+
+//go:linkname _${sym} ${sym}
+var _${sym} byte
+var syso__${sym} = &sysoCaller{ptr: unsafe.Pointer(&_${sym})}
+
+EOF
+done