Translate paths in compilepkg output.

When cgo2 is involved in a build, the go srcs are linked/copied
into a temporary directory to keep everything in the same
directory. Those paths are unrecognizable to IDEs, being absolute
paths outside the project directory.

This adds a translation step to replace paths in the compiler output,
translating them back to the input paths that were given to the tool
in the first place.

This doesn't affect -trimpath, which affects runtime error messages
but not compilation-time error messages; for the same reason, this
change can't use that flag to do its dirty work.

Fixes cockroachdb/cockroach#76377.

Release note: None

Author: rickystewart

go/tools/builders/cgo2.go

@@ -33,7 +33,7 @@ import (
 )
 
 // cgo2 processes a set of mixed source files with cgo.
-func cgo2(goenv *env, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs []string, packagePath, packageName string, cc string, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags []string, cgoExportHPath string, cgoGoSrcsPath string) (srcDir string, allGoSrcs, cObjs []string, err error) {
+func cgo2(goenv *env, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs []string, packagePath, packageName string, cc string, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags []string, cgoExportHPath string, cgoGoSrcsPath string) (allGoSrcsDir string, allGoSrcs []pathPair, cObjs []string, err error) {
 	// Report an error if the C/C++ toolchain wasn't configured.
 	if cc == "" {
 		err := cgoError(cgoSrcs[:])
@@ -132,7 +132,7 @@ func cgo2(goenv *env, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSr
 
 	// If cgo sources are in different directories, gather them into a temporary
 	// directory so we can use -srcdir.
-	srcDir = filepath.Dir(cgoSrcs[0])
+	srcDir := filepath.Dir(cgoSrcs[0])
 	srcsInSingleDir := true
 	for _, src := range cgoSrcs[1:] {
 		if filepath.Dir(src) != srcDir {
@@ -199,13 +199,16 @@ func cgo2(goenv *env, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSr
 	// Note: The tools/gopackagesdriver will break if the naming convention
 	// changes for genGoSrcs files. Changes here should be reflected
 	// there.
-	genGoSrcs := make([]string, 1+len(cgoSrcs))
-	genGoSrcs[0] = filepath.Join(workDir, "_cgo_gotypes.go")
+	// genGoSrcs contains the go source files generated by the cgo compiler plus 2 additional files, one at each end:
+	// genGoSrcs[0] is the gotypes file, while genGoSrcs[-1] is the imports file.
+	genGoSrcs := make([]pathPair, 2+len(cgoSrcs))
+	genGoSrcs[0].workingPath = filepath.Join(workDir, "_cgo_gotypes.go")
 	genCSrcs := make([]string, 1+len(cgoSrcs))
 	genCSrcs[0] = filepath.Join(workDir, "_cgo_export.c")
 	for i, src := range cgoSrcs {
 		stem := strings.TrimSuffix(filepath.Base(src), ".go")
-		genGoSrcs[i+1] = filepath.Join(workDir, stem+".cgo1.go")
+		genGoSrcs[i+1].inputPath = src
+		genGoSrcs[i+1].workingPath = filepath.Join(workDir, stem+".cgo1.go")
 		genCSrcs[i+1] = filepath.Join(workDir, stem+".cgo2.c")
 	}
 	cgoMainC := filepath.Join(workDir, "_cgo_main.c")
@@ -279,10 +282,10 @@ func cgo2(goenv *env, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSr
 	if err := goenv.runCommand(args); err != nil {
 		return "", nil, nil, err
 	}
-	genGoSrcs = append(genGoSrcs, cgoImportsGo)
+	genGoSrcs[len(genGoSrcs)-1].workingPath = cgoImportsGo
 	if cgoGoSrcsPath != "" {
 		for _, src := range genGoSrcs {
-			if err := copyFile(src, filepath.Join(cgoGoSrcsPath, filepath.Base(src))); err != nil {
+			if err := copyFile(src.workingPath, filepath.Join(cgoGoSrcsPath, filepath.Base(src.workingPath))); err != nil {
 				return "", nil, nil, err
 			}
 		}
@@ -295,9 +298,10 @@ func cgo2(goenv *env, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSr
 		return "", nil, nil, err
 	}
 
-	allGoSrcs = make([]string, len(goSrcs)+len(genGoSrcs))
+	allGoSrcs = make([]pathPair, len(goSrcs)+len(genGoSrcs))
 	for i := range goSrcs {
-		allGoSrcs[i] = filepath.Join(workDir, goBases[i])
+		allGoSrcs[i].inputPath = goSrcs[i]
+		allGoSrcs[i].workingPath = filepath.Join(workDir, goBases[i])
 	}
 	copy(allGoSrcs[len(goSrcs):], genGoSrcs)
 	return workDir, allGoSrcs, cObjs, nil

go/tools/builders/compilepkg.go

@@ -329,15 +329,20 @@ func compileArchive(
 	// If we have cgo, generate separate C and go files, and compile the
 	// C files.
 	var objFiles []string
+	var goSrcsMapping []pathPair
 	if compilingWithCgo {
 		var srcDir string
 		if coverMode != "" && cgoGoSrcsForNogoPath != "" {
 			// If the package uses Cgo, compile .s and .S files with cgo2, not the Go assembler.
 			// Otherwise: the .s/.S files will be compiled with the Go assembler later
-			srcDir, goSrcs, objFiles, err = cgo2(goenv, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs, packagePath, packageName, cc, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags, cgoExportHPath, "")
+			srcDir, goSrcsMapping, objFiles, err = cgo2(goenv, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs, packagePath, packageName, cc, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags, cgoExportHPath, "")
 			if err != nil {
 				return err
 			}
+			goSrcs = make([]string, len(goSrcsMapping))
+			for i, v := range goSrcsMapping {
+				goSrcs[i] = v.workingPath
+			}
 			// Also run cgo on original source files, not coverage instrumented, if using nogo.
 			// The compilation outputs are only used to run cgo, but the generated sources are
 			// passed to the separate nogo action via cgoGoSrcsForNogoPath.
@@ -348,10 +353,14 @@ func compileArchive(
 		} else {
 			// If the package uses Cgo, compile .s and .S files with cgo2, not the Go assembler.
 			// Otherwise: the .s/.S files will be compiled with the Go assembler later
-			srcDir, goSrcs, objFiles, err = cgo2(goenv, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs, packagePath, packageName, cc, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags, cgoExportHPath, cgoGoSrcsForNogoPath)
+			srcDir, goSrcsMapping, objFiles, err = cgo2(goenv, goSrcs, cgoSrcs, cSrcs, cxxSrcs, objcSrcs, objcxxSrcs, sSrcs, hSrcs, packagePath, packageName, cc, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags, cgoExportHPath, cgoGoSrcsForNogoPath)
 			if err != nil {
 				return err
 			}
+			goSrcs = make([]string, len(goSrcsMapping))
+			for i, v := range goSrcsMapping {
+				goSrcs[i] = v.workingPath
+			}
 		}
 		gcFlags = append(gcFlags, createTrimPath(gcFlags, srcDir))
 	} else {
@@ -421,7 +430,7 @@ func compileArchive(
 	}
 
 	// Compile the filtered .go files.
-	if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath, gcFlags, pgoprofile, outLinkObj, outInterfacePath, coverageCfg); err != nil {
+	if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath, gcFlags, goSrcsMapping, pgoprofile, outLinkObj, outInterfacePath, coverageCfg); err != nil {
 		return err
 	}
 
@@ -508,7 +517,7 @@ func checkImportsAndBuildCfg(goenv *env, importPath string, srcs archiveSrcs, de
 	return importcfgPath, nil
 }
 
-func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath string, gcFlags []string, pgoprofile, outLinkobjPath, outInterfacePath, coverageCfg string) error {
+func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath string, gcFlags []string, paths []pathPair, pgoprofile, outLinkobjPath, outInterfacePath, coverageCfg string) error {
 	args := goenv.goTool("compile")
 	args = append(args, "-p", packagePath, "-importcfg", importcfgPath, "-pack")
 	if embedcfgPath != "" {
@@ -532,7 +541,7 @@ func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPa
 	args = append(args, "--")
 	args = append(args, srcs...)
 	absArgs(args, []string{"-I", "-o", "-trimpath", "-importcfg"})
-	return goenv.runCommand(args)
+	return goenv.runCommandAndReplacePaths(args, paths)
 }
 
 func appendToArchive(goenv *env, pack, outPath string, objFiles []string) error {

go/tools/builders/env.go

@@ -68,6 +68,17 @@ type env struct {
 	shouldPreserveWorkDir bool
 }
 
+// pathPair maps the input path given to the builder to the
+// working path, usually in a temporary directory, given to
+// tools like cgo2.
+type pathPair struct {
+	// inputPath is the path given to the builder.
+	// Files generated by the builder will have an inputPath of "".
+	inputPath string
+	// workingPath is the path given to the tool itself.
+	workingPath string
+}
+
 // envFlags registers flags common to multiple builders and returns an env
 // configured with those flags.
 func envFlags(flags *flag.FlagSet) *env {
@@ -141,7 +152,8 @@ func (e *env) goCmd(cmd string, args ...string) []string {
 }
 
 // runCommand executes a subprocess that inherits stdout, stderr, and the
-// environment from this process.
+// environment from this process. Paths under the working directory will
+// be relativized in the output.
 func (e *env) runCommand(args []string) error {
 	cmd := exec.Command(args[0], args[1:]...)
 	// Redirecting stdout to stderr. This mirrors behavior in the go command:
@@ -154,6 +166,22 @@ func (e *env) runCommand(args []string) error {
 	return err
 }
 
+// runCommandAndReplacePaths executes a subprocess that inherits stdout,
+// stderr, and the environment from this process. The stderr stream
+// will be filtered to replace the given pairs of paths, and then paths
+// under the working directory will be relativized in the output.
+func (e *env) runCommandAndReplacePaths(args []string, paths []pathPair) error {
+	cmd := exec.Command(args[0], args[1:]...)
+	// Redirecting stdout to stderr. This mirrors behavior in the go command:
+	// https://go.googlesource.com/go/+/refs/tags/go1.15.2/src/cmd/go/internal/work/exec.go#1958
+	buf := &bytes.Buffer{}
+	cmd.Stdout = buf
+	cmd.Stderr = buf
+	err := runAndLogCommand(cmd, e.verbose)
+	os.Stderr.Write(relativizePaths(replacePaths(paths, buf.Bytes())))
+	return err
+}
+
 // runCommandToFile executes a subprocess and writes stdout/stderr to the given
 // writers.
 func (e *env) runCommandToFile(out, err io.Writer, args []string) error {
@@ -480,6 +508,17 @@ func relativizePaths(output []byte) []byte {
 	return bytes.ReplaceAll(output, dirBytes, []byte{'.'})
 }
 
+// replacePaths converts any instances in data of the destPath of any
+// element of the paths to the corresponding sourcePath.
+func replacePaths(paths []pathPair, data []byte) []byte {
+	for _, pair := range paths {
+		if pair.inputPath != "" && pair.workingPath != pair.inputPath {
+			data = bytes.ReplaceAll(data, []byte(pair.workingPath), []byte(pair.inputPath))
+		}
+	}
+	return data
+}
+
 // formatCommand formats cmd as a string that can be pasted into a shell.
 // Spaces in environment variables and arguments are escaped as needed.
 func formatCommand(cmd *exec.Cmd) string {