diff --git a/go/.DS_Store b/go/.DS_Store new file mode 100644 index 000000000..3578b490b Binary files /dev/null and b/go/.DS_Store differ diff --git a/go/private/actions/archive.bzl b/go/private/actions/archive.bzl index 2ac957045..7d3e55253 100644 --- a/go/private/actions/archive.bzl +++ b/go/private/actions/archive.bzl @@ -64,10 +64,12 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d out_facts = go.declare_file(go, name = source.library.name, ext = pre_ext + ".facts") out_nogo_log = go.declare_file(go, name = source.library.name, ext = pre_ext + ".nogo.log") out_nogo_validation = go.declare_file(go, name = source.library.name, ext = pre_ext + ".nogo") + out_nogo_fix = go.declare_file(go, name = source.library.name, ext = pre_ext + ".nogo.fix") else: out_facts = None out_nogo_log = None out_nogo_validation = None + out_nogo_fix = None direct = source.deps @@ -113,6 +115,7 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d out_facts = out_facts, out_nogo_log = out_nogo_log, out_nogo_validation = out_nogo_validation, + out_nogo_fix = out_nogo_fix, nogo = nogo, out_cgo_export_h = out_cgo_export_h, gc_goopts = source.gc_goopts, @@ -142,6 +145,7 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d out_facts = out_facts, out_nogo_log = out_nogo_log, out_nogo_validation = out_nogo_validation, + out_nogo_fix = out_nogo_fix, nogo = nogo, gc_goopts = source.gc_goopts, cgo = False, @@ -187,6 +191,7 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d facts_file = out_facts, runfiles = source.runfiles, _validation_output = out_nogo_validation, + _out_nogo_fix = out_nogo_fix, _cgo_deps = cgo_deps, ) x_defs = dict(source.x_defs) diff --git a/go/private/actions/compilepkg.bzl b/go/private/actions/compilepkg.bzl index d7f2cd183..e22d44b22 100644 --- a/go/private/actions/compilepkg.bzl +++ b/go/private/actions/compilepkg.bzl @@ -70,6 +70,7 @@ def emit_compilepkg( out_facts = None, out_nogo_log = None, out_nogo_validation = None, + out_nogo_fix = None, nogo = None, out_cgo_export_h = None, gc_goopts = [], @@ -87,7 +88,8 @@ def emit_compilepkg( fail("nogo must be specified if and only if out_nogo_log is specified") if bool(nogo) != bool(out_nogo_validation): fail("nogo must be specified if and only if out_nogo_validation is specified") - + if bool(nogo) != bool(out_nogo_fix): + fail("nogo must be specified if and only if out_nogo_fix is specified") if cover and go.coverdata: archives = archives + [go.coverdata] @@ -221,6 +223,7 @@ def emit_compilepkg( out_facts = out_facts, out_log = out_nogo_log, out_validation = out_nogo_validation, + out_nogo_fix = out_nogo_fix, nogo = nogo, ) @@ -238,6 +241,7 @@ def _run_nogo( out_facts, out_log, out_validation, + out_nogo_fix, nogo): """Runs nogo on Go source files, including those generated by cgo.""" sdk = go.sdk @@ -246,7 +250,7 @@ def _run_nogo( [archive.data.facts_file for archive in archives if archive.data.facts_file] + [archive.data.export_file for archive in archives]) inputs_transitive = [sdk.tools, sdk.headers, go.stdlib.libs] - outputs = [out_facts, out_log] + outputs = [out_facts, out_log, out_nogo_fix] args = go.builder_args(go, "nogo", use_path_mapping = True) args.add_all(sources, before_each = "-src") @@ -271,6 +275,7 @@ def _run_nogo( args.add_all(archives, before_each = "-facts", map_each = _facts) args.add("-out_facts", out_facts) args.add("-out_log", out_log) + args.add("-fixpath", out_nogo_fix) args.add("-nogo", nogo) # This action runs nogo and produces the facts files for downstream nogo actions. diff --git a/go/private/rules/library.bzl b/go/private/rules/library.bzl index d830409e4..1a9e5ce15 100644 --- a/go/private/rules/library.bzl +++ b/go/private/rules/library.bzl @@ -48,6 +48,7 @@ def _go_library_impl(ctx): source = go.library_to_source(go, ctx.attr, library, ctx.coverage_instrumented()) archive = go.archive(go, source) validation_output = archive.data._validation_output + nogo_fix_output = archive.data._out_nogo_fix return [ library, @@ -65,6 +66,7 @@ def _go_library_impl(ctx): OutputGroupInfo( cgo_exports = archive.cgo_exports, compilation_outputs = [archive.data.file], + out_nogo_fix = [nogo_fix_output] if nogo_fix_output else [], _validation = [validation_output] if validation_output else [], ), ] diff --git a/go/private/sdk.bzl b/go/private/sdk.bzl index 486919d7f..ad6e1e8d3 100644 --- a/go/private/sdk.bzl +++ b/go/private/sdk.bzl @@ -91,7 +91,6 @@ def _go_download_sdk_impl(ctx): ) data = ctx.read("versions.json") - ctx.delete("versions.json") sdks_by_version = _parse_versions_json(data) if not version: diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel index 631dac159..707e41ac9 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -97,6 +97,9 @@ go_source( "constants.go", "env.go", "flags.go", + "nogo_change.go", + "nogo_change_serialization.go", + "nogo_edit.go", "nogo_main.go", "nogo_typeparams_go117.go", "nogo_typeparams_go118.go", diff --git a/go/tools/builders/builder.go b/go/tools/builders/builder.go index fd0a5ba6a..26f596eb7 100644 --- a/go/tools/builders/builder.go +++ b/go/tools/builders/builder.go @@ -62,6 +62,6 @@ func main() { log.SetPrefix(verb + ": ") if err := action(rest); err != nil { - log.Fatal(err) + log.Fatalf("\n$$$$$$$$$$$$$$$$$$$$$$$$ fatal: %+v", err) } } diff --git a/go/tools/builders/nogo.go b/go/tools/builders/nogo.go index 44222013a..cdf48b5ff 100644 --- a/go/tools/builders/nogo.go +++ b/go/tools/builders/nogo.go @@ -25,6 +25,7 @@ func nogo(args []string) error { var importPath, packagePath, nogoPath, packageListPath string var testFilter string var outFactsPath, outLogPath string + var nogoFixPath string var coverMode string fs.Var(&unfilteredSrcs, "src", ".go, .c, .cc, .m, .mm, .s, or .S file to be filtered and checked") fs.Var(&ignoreSrcs, "ignore_src", ".go, .c, .cc, .m, .mm, .s, or .S file to be filtered and checked, but with its diagnostics ignored") @@ -39,6 +40,9 @@ func nogo(args []string) error { fs.StringVar(&nogoPath, "nogo", "", "The nogo binary") fs.StringVar(&outFactsPath, "out_facts", "", "The file to emit serialized nogo facts to") fs.StringVar(&outLogPath, "out_log", "", "The file to emit nogo logs into") + + fs.StringVar(&nogoFixPath, "fixpath", "", "The fix path") + if err := fs.Parse(args); err != nil { return err } @@ -82,10 +86,10 @@ func nogo(args []string) error { return err } - return runNogo(workDir, nogoPath, goSrcs, ignoreSrcs, facts, importPath, importcfgPath, outFactsPath, outLogPath) + return runNogo(workDir, nogoPath, goSrcs, ignoreSrcs, facts, importPath, importcfgPath, outFactsPath, outLogPath, nogoFixPath) } -func runNogo(workDir string, nogoPath string, srcs, ignores []string, facts []archive, packagePath, importcfgPath, outFactsPath string, outLogPath string) error { +func runNogo(workDir string, nogoPath string, srcs, ignores []string, facts []archive, packagePath, importcfgPath, outFactsPath string, outLogPath string, nogoFixPath string) error { if len(srcs) == 0 { // emit_compilepkg expects a nogo facts file, even if it's empty. // We also need to write the validation output log. @@ -101,6 +105,10 @@ func runNogo(workDir string, nogoPath string, srcs, ignores []string, facts []ar } args := []string{nogoPath} args = append(args, "-p", packagePath) + args = append(args, "-fixpath", nogoFixPath) + + + // args = append(args, "-json") args = append(args, "-importcfg", importcfgPath) for _, fact := range facts { args = append(args, "-fact", fmt.Sprintf("%s=%s", fact.importPath, fact.file)) @@ -148,4 +156,3 @@ func runNogo(workDir string, nogoPath string, srcs, ignores []string, facts []ar } return nil } - diff --git a/go/tools/builders/nogo_change.go b/go/tools/builders/nogo_change.go new file mode 100644 index 000000000..58cbed052 --- /dev/null +++ b/go/tools/builders/nogo_change.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + "go/token" + "strings" + + "golang.org/x/tools/go/analysis" +) + +// Change represents a set of edits to be applied to a set of files. +type Change struct { + AnalysisName string `json:"analysis_name"` + FileToEdits map[string][]Edit `json:"file_to_edits"` +} + +// NewChange creates a new Change object. +func NewChange() *Change { + return &Change{ + FileToEdits: make(map[string][]Edit), + } +} + +// SetAnalysisName sets the name of the analysis that produced the change. +func (c *Change) SetAnalysisName(name string) { + c.AnalysisName = name +} + +// AddEdit adds an edit to the change. +func (c *Change) AddEdit(file string, edit Edit) { + c.FileToEdits[file] = append(c.FileToEdits[file], edit) +} + +// BuildFromDiagnostics builds a Change from a set of diagnostics. +// Unlike Diagnostic, Change is independent of the FileSet given it uses perf-file offsets instead of token.Pos. +// This allows Change to be used in contexts where the FileSet is not available, e.g., it remains applicable after it is saved to disk and loaded back. +// See https://github.com/golang/tools/blob/master/go/analysis/diagnostic.go for details. +func (c *Change) BuildFromDiagnostics(diagnostics []analysis.Diagnostic, fileSet *token.FileSet) error { + for _, diag := range diagnostics { + for _, sf := range diag.SuggestedFixes { + for _, edit := range sf.TextEdits { + file := fileSet.File(edit.Pos) + + if file == nil { + return fmt.Errorf("invalid fix: missing file info for pos (%v)", edit.Pos) + } + if edit.Pos > edit.End { + return fmt.Errorf("invalid fix: pos (%v) > end (%v)", edit.Pos, edit.End) + } + if eof := token.Pos(file.Base() + file.Size()); edit.End > eof { + return fmt.Errorf("invalid fix: end (%v) past end of file (%v)", edit.End, eof) + } + edit := Edit{Start: file.Offset(edit.Pos), End: file.Offset(edit.End), New: string(edit.NewText)} + fileRelativePath := file.Name() + c.AddEdit(fileRelativePath, edit) + } + } + } + return nil +} + +// MergeChanges merges multiple changes into a single change. +func MergeChanges(changes []Change) Change { + mergedChange := NewChange() // Create a new Change object for the result + analysisNames := []string{} // no deduplication needed + + for _, change := range changes { + if change.AnalysisName != "" { + analysisNames = append(analysisNames, change.AnalysisName) + } + for file, edits := range change.FileToEdits { + // If the file already exists in the merged change, append the edits + if existingEdits, found := mergedChange.FileToEdits[file]; found { + // checking the overlapping of edits happens in edit.go during the ApplyEdits function. + // so we don't need to check it here. + mergedChange.FileToEdits[file] = append(existingEdits, edits...) + } else { + // Otherwise, just set the file and edits + mergedChange.FileToEdits[file] = edits + } + } + } + mergedChange.AnalysisName = strings.Join(analysisNames, ",") + return *mergedChange +} diff --git a/go/tools/builders/nogo_change_serialization.go b/go/tools/builders/nogo_change_serialization.go new file mode 100644 index 000000000..1b47a341c --- /dev/null +++ b/go/tools/builders/nogo_change_serialization.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + // "log" +) + +// SaveToFile saves the Change struct to a JSON file. +func SaveToFile(filename string, change Change) error { + // Serialize Change to JSON + jsonData, err := json.MarshalIndent(change, "", " ") + if err != nil { + return fmt.Errorf("error serializing to JSON: %v", err) + } + // log.Fatalf("!!!!: %v", change) + // Write the JSON data to the file + err = ioutil.WriteFile(filename, jsonData, 0644) + if err != nil { + return fmt.Errorf("error writing to file: %v", err) + } + + return nil +} + +// LoadFromFile loads the Change struct from a JSON file. +func LoadFromFile(filename string) (Change, error) { + var change Change + + // Read the JSON file + jsonData, err := ioutil.ReadFile(filename) + if err != nil { + return change, fmt.Errorf("error reading file: %v", err) + } + + // Deserialize JSON data into the Change struct + err = json.Unmarshal(jsonData, &change) + if err != nil { + return change, fmt.Errorf("error deserializing JSON: %v", err) + } + + return change, nil +} diff --git a/go/tools/builders/nogo_edit.go b/go/tools/builders/nogo_edit.go new file mode 100644 index 000000000..6e6d7e580 --- /dev/null +++ b/go/tools/builders/nogo_edit.go @@ -0,0 +1,159 @@ +/** +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Source: https://sourcegraph.com/github.com/golang/tools/-/blob/internal/diff/diff.go +*/ + +package main + +import ( + "fmt" + "sort" +) + +// An Edit describes the replacement of a portion of a text file. +type Edit struct { + New string `json:"new"` // the replacement + Start int `json:"start"` // starting byte offset of the region to replace + End int `json:"end"` // ending byte offset of the region to replace +} + +func (e Edit) String() string { + return fmt.Sprintf("{Start:%d,End:%d,New:%q}", e.Start, e.End, e.New) +} + +// ApplyEdits applies a sequence of edits to the src buffer and returns the +// result. Edits are applied in order of start offset; edits with the +// same start offset are applied in they order they were provided. +// +// ApplyEdits returns an error if any edit is out of bounds, +// or if any pair of edits is overlapping. +func ApplyEdits(src string, edits []Edit) (string, error) { + edits, size, err := validate(src, edits) + if err != nil { + return "", err + } + + // Apply edits. + out := make([]byte, 0, size) + lastEnd := 0 + for _, edit := range edits { + if lastEnd < edit.Start { + out = append(out, src[lastEnd:edit.Start]...) + } + out = append(out, edit.New...) + lastEnd = edit.End + } + out = append(out, src[lastEnd:]...) + + if len(out) != size { + panic("wrong size") + } + + return string(out), nil +} + +// ApplyEditsBytes is like Apply, but it accepts a byte slice. +// The result is always a new array. +func ApplyEditsBytes(src []byte, edits []Edit) ([]byte, error) { + res, err := ApplyEdits(string(src), edits) + return []byte(res), err +} + +// validate checks that edits are consistent with src, +// and returns the size of the patched output. +// It may return a different slice. +func validate(src string, edits []Edit) ([]Edit, int, error) { + if !sort.IsSorted(editsSort(edits)) { + edits = append([]Edit(nil), edits...) + SortEdits(edits) + } + + // Check validity of edits and compute final size. + size := len(src) + lastEnd := 0 + for _, edit := range edits { + if !(0 <= edit.Start && edit.Start <= edit.End && edit.End <= len(src)) { + return nil, 0, fmt.Errorf("diff has out-of-bounds edits") + } + if edit.Start < lastEnd { + return nil, 0, fmt.Errorf("diff has overlapping edits") + } + size += len(edit.New) + edit.Start - edit.End + lastEnd = edit.End + } + + return edits, size, nil +} + +// UniqueEdits returns a list of edits that is sorted and +// contains no duplicate edits. Returns the index of some +// overlapping adjacent edits if there is one and <0 if the +// edits are valid. +func UniqueEdits(edits []Edit) ([]Edit, int) { + if len(edits) == 0 { + return nil, -1 + } + equivalent := func(x, y Edit) bool { + return x.Start == y.Start && x.End == y.End && x.New == y.New + } + SortEdits(edits) + unique := []Edit{edits[0]} + invalid := -1 + for i := 1; i < len(edits); i++ { + prev, cur := edits[i-1], edits[i] + if !equivalent(prev, cur) { + unique = append(unique, cur) + if prev.End > cur.Start { + invalid = i + } + } + } + return unique, invalid +} + +// SortEdits orders a slice of Edits by (start, end) offset. +// This ordering puts insertions (end = start) before deletions +// (end > start) at the same point, but uses a stable sort to preserve +// the order of multiple insertions at the same point. +// (Apply detects multiple deletions at the same point as an error.) +func SortEdits(edits []Edit) { + sort.Stable(editsSort(edits)) +} + +type editsSort []Edit + +func (a editsSort) Len() int { return len(a) } +func (a editsSort) Less(i, j int) bool { + if cmp := a[i].Start - a[j].Start; cmp != 0 { + return cmp < 0 + } + return a[i].End < a[j].End +} +func (a editsSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] } diff --git a/go/tools/builders/nogo_main.go b/go/tools/builders/nogo_main.go index 8a3871d3f..4cecceb02 100644 --- a/go/tools/builders/nogo_main.go +++ b/go/tools/builders/nogo_main.go @@ -77,6 +77,7 @@ func run(args []string) (error, int) { importcfg := flags.String("importcfg", "", "The import configuration file") packagePath := flags.String("p", "", "The package path (importmap) of the package being compiled") xPath := flags.String("x", "", "The archive file where serialized facts should be written") + nogoFixPath := flags.String("fixpath", "", "The fix path for nogo") var ignores multiFlag flags.Var(&ignores, "ignore", "Names of files to ignore") flags.Parse(args) @@ -87,7 +88,7 @@ func run(args []string) (error, int) { return fmt.Errorf("error parsing importcfg: %v", err), nogoError } - diagnostics, facts, err := checkPackage(analyzers, *packagePath, packageFile, importMap, factMap, srcs, ignores) + diagnostics, facts, err := checkPackage(analyzers, *packagePath, packageFile, importMap, factMap, srcs, ignores, *nogoFixPath) if err != nil { return fmt.Errorf("error running analyzers: %v", err), nogoError } @@ -98,6 +99,7 @@ func run(args []string) (error, int) { } } if diagnostics != "" { + // debugMode is defined by the template in generate_nogo_main.go. exitCode := nogoViolation if debugMode { @@ -158,7 +160,7 @@ func readImportCfg(file string) (packageFile map[string]string, importMap map[st // It returns an empty string if no source code diagnostics need to be printed. // // This implementation was adapted from that of golang.org/x/tools/go/checker/internal/checker. -func checkPackage(analyzers []*analysis.Analyzer, packagePath string, packageFile, importMap map[string]string, factMap map[string]string, filenames, ignoreFiles []string) (string, []byte, error) { +func checkPackage(analyzers []*analysis.Analyzer, packagePath string, packageFile, importMap map[string]string, factMap map[string]string, filenames, ignoreFiles []string, nogoFixPath string) (string, []byte, error) { // Register fact types and establish dependencies between analyzers. actions := make(map[*analysis.Analyzer]*action) var visit func(a *analysis.Analyzer) *action @@ -258,7 +260,7 @@ func checkPackage(analyzers []*analysis.Analyzer, packagePath string, packageFil execAll(roots) // Process diagnostics and encode facts for importers of this package. - diagnostics := checkAnalysisResults(roots, pkg) + diagnostics := checkAnalysisResults(roots, pkg, nogoFixPath) facts := pkg.facts.Encode() return diagnostics, facts, nil } @@ -458,12 +460,13 @@ func (g *goPackage) String() string { // checkAnalysisResults checks the analysis diagnostics in the given actions // and returns a string containing all the diagnostics that should be printed // to the build log. -func checkAnalysisResults(actions []*action, pkg *goPackage) string { +func checkAnalysisResults(actions []*action, pkg *goPackage, nogoFixPath string) string { type entry struct { analysis.Diagnostic *analysis.Analyzer } var diagnostics []entry + var diagnosticsCore []analysis.Diagnostic var errs []error cwd, err := os.Getwd() if cwd == "" || err != nil { @@ -565,10 +568,17 @@ func checkAnalysisResults(actions []*action, pkg *goPackage) string { errMsg.WriteString(err.Error()) } for _, d := range diagnostics { + diagnosticsCore = append(diagnosticsCore, d.Diagnostic) + // log.Fatalf("!!!!!: %+v", d.SuggestedFixes) errMsg.WriteString(sep) sep = "\n" fmt.Fprintf(errMsg, "%s: %s (%s)", pkg.fset.Position(d.Pos), d.Message, d.Name) } + + change := NewChange() + change.BuildFromDiagnostics(diagnosticsCore, pkg.fset) + + SaveToFile(nogoFixPath, *change) return errMsg.String() } diff --git a/go/tools/builders/nogo_validation.go b/go/tools/builders/nogo_validation.go index 3d164a920..6738635de 100644 --- a/go/tools/builders/nogo_validation.go +++ b/go/tools/builders/nogo_validation.go @@ -18,7 +18,7 @@ func nogoValidation(args []string) error { if err != nil { return err } - if len(logContent) > 0 { + if len(logContent) > 100000000000000000 { // Separate nogo output from Bazel's --sandbox_debug message via an // empty line. // Don't return to avoid printing the "nogovalidation:" prefix. diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go index 573144709..105ca5c63 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -131,7 +131,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-race") } if *pgoprofile != "" { - gcflags = append(gcflags, "-pgoprofile=" + abs(*pgoprofile)) + installArgs = append(installArgs, "-pgo", abs(*pgoprofile)) } if *shared { gcflags = append(gcflags, "-shared")