unix: generate all Linux go files from source

Right now the process for adding in new constants, errors, or syscalls
for Linux is a pain and unreliable. The scripts are designed to be run
on the target architecture and use the header files installed on the
user's system. This makes it hard to generate files for all the
architectures or to have consistency between users. See golang/go#15282.

This CL fixes this issue by making all of the files for the 11 supported
architectures directly from source checkouts of Linux, glibc, and bluez.
This is done using Docker, the gcc cross-compilers, and qemu emulation.
Previously discussed here:
    https://go-review.googlesource.com/c/37589/

A README.md file is also added to explain how all the parts of the build
system work.

In order to get the build working for all the architectures, I made
some changes to the other scripts called from mkall_linux.go:
  - Files only used for generating linux code, moved to linux/
  - linux/mksysnum.pl supports a specified CC compiler.
  - The generated C code in mkerrors.sh changed to avoid a warning
  - mkerrors.sh headers changed to fix powerpc64 bug in sys/ioctl.h
  - linux/types.go no longer needs to export Ptrace structs in lowercase

Build instructions:
  - Host system needs to be x86-64 Linux
  - Install Docker (https://docs.docker.com/engine/installation/)
  - ./mkall.sh (That's it!!!)

Change-Id: I87067c14442ba12f8d51991349a43a9d73f38ae0
Reviewed-on: https://go-review.googlesource.com/37943
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

unix/mkpost.go

@@ -8,10 +8,11 @@
 // modify the generated types. It is used to clean up
 // the sys API in an architecture specific manner.
 //
-// mkpost is run after cgo -godefs by mkall.sh.
+// mkpost is run after cgo -godefs; see README.md.
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"go/format"
 	"io/ioutil"
@@ -21,42 +22,67 @@ import (
 )
 
 func main() {
+	// Get the OS and architecture (using GOARCH_TARGET if it exists)
+	goos := os.Getenv("GOOS")
+	goarch := os.Getenv("GOARCH_TARGET")
+	if goarch == "" {
+		goarch = os.Getenv("GOARCH")
+	}
+	// Check that we are using the new build system if we should be.
+	if goos == "linux" && goarch != "sparc64" {
+		if os.Getenv("GOLANG_SYS_BUILD") != "docker" {
+			os.Stderr.WriteString("In the new build system, mkpost should not be called directly.\n")
+			os.Stderr.WriteString("See README.md\n")
+			os.Exit(1)
+		}
+	}
+
 	b, err := ioutil.ReadAll(os.Stdin)
 	if err != nil {
 		log.Fatal(err)
 	}
-	s := string(b)
 
-	goarch := os.Getenv("GOARCH")
-	goos := os.Getenv("GOOS")
+	// If we have empty Ptrace structs, we should delete them. Only s390x emits
+	// nonempty Ptrace structs.
+	ptraceRexexp := regexp.MustCompile(`type Ptrace((Psw|Fpregs|Per) struct {\s*})`)
+	b = ptraceRexexp.ReplaceAll(b, nil)
+
+	// Replace the control_regs union with a blank identifier for now.
+	controlRegsRegex := regexp.MustCompile(`(Control_regs)\s+\[0\]uint64`)
+	b = controlRegsRegex.ReplaceAll(b, []byte("_ [0]uint64"))
+
+	// Remove fields that are added by glibc
+	// Note that this is unstable as the identifers are private.
+	removeFieldsRegex := regexp.MustCompile(`X__glibc\S*`)
+	b = removeFieldsRegex.ReplaceAll(b, []byte("_"))
+
+	// We refuse to export private fields on s390x
 	if goarch == "s390x" && goos == "linux" {
-		// Export the types of PtraceRegs fields.
-		re := regexp.MustCompile("ptrace(Psw|Fpregs|Per)")
-		s = re.ReplaceAllString(s, "Ptrace$1")
+		// Remove cgo padding fields
+		removeFieldsRegex := regexp.MustCompile(`Pad_cgo_\d+`)
+		b = removeFieldsRegex.ReplaceAll(b, []byte("_"))
 
-		// Replace padding fields inserted by cgo with blank identifiers.
-		re = regexp.MustCompile("Pad_cgo[A-Za-z0-9_]*")
-		s = re.ReplaceAllString(s, "_")
-
-		// Replace other unwanted fields with blank identifiers.
-		re = regexp.MustCompile("X_[A-Za-z0-9_]*")
-		s = re.ReplaceAllString(s, "_")
-
-		// Replace the control_regs union with a blank identifier for now.
-		re = regexp.MustCompile("(Control_regs)\\s+\\[0\\]uint64")
-		s = re.ReplaceAllString(s, "_ [0]uint64")
+		// Remove padding, hidden, or unused fields
+		removeFieldsRegex = regexp.MustCompile(`X_\S+`)
+		b = removeFieldsRegex.ReplaceAll(b, []byte("_"))
 	}
 
+	// Remove the first line of warning from cgo
+	b = b[bytes.IndexByte(b, '\n')+1:]
+	// Modify the command in the header to include:
+	//  mkpost, our own warning, and a build tag.
+	replacement := fmt.Sprintf(`$1 | go run mkpost.go
+// Code generated by the command above; see README.md. DO NOT EDIT.
+
+// +build %s,%s`, goarch, goos)
+	cgoCommandRegex := regexp.MustCompile(`(cgo -godefs .*)`)
+	b = cgoCommandRegex.ReplaceAll(b, []byte(replacement))
+
 	// gofmt
-	b, err = format.Source([]byte(s))
+	b, err = format.Source(b)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// Append this command to the header to show where the new file
-	// came from.
-	re := regexp.MustCompile("(cgo -godefs [a-zA-Z0-9_]+\\.go.*)")
-	b = re.ReplaceAll(b, []byte("$1 | go run mkpost.go"))
-
-	fmt.Printf("%s", b)
+	os.Stdout.Write(b)
 }