feat: support gc root mask to avoid dwarf search missing memory (#26)

* feat: support stackmap and gcbss/datamask to avoid dwarf search missing memory

* feat: use conservative scanning

* feat: adjust log level

README.md

@@ -49,7 +49,7 @@ successfully output to `grf.out`
 
 ## Docs
 
-[Principle](docs/principle.md) | [实现原理](docs/principle_cn.md)
+[How it Works](docs/principle.md) | [实现原理](docs/principle_cn.md)
 
 ## Credit
 

cmd/grf/cmds/commands.go

@@ -36,6 +36,9 @@ var (
 	conf        *config.Config
 	loadConfErr error
 	outFile     string
+
+	// verbose is whether to log verbose info, like debug logs.
+	verbose bool
 )
 
 // New returns an initialized command tree.
@@ -89,21 +92,21 @@ You'll have to wait for goref until it outputs 'successfully output to ...', or
 	coreCommand.Flags().StringVarP(&outFile, "out", "o", "grf.out", "output file name")
 	rootCommand.AddCommand(coreCommand)
 
-	versionVerbose := false
 	versionCommand := &cobra.Command{
 		Use:   "version",
 		Short: "Prints version.",
 		Run: func(cmd *cobra.Command, args []string) {
 			fmt.Printf("Goref Tool\n%s\n", version.DelveVersion)
-			if versionVerbose {
+			if verbose {
 				fmt.Printf("Build Details: %s\n", version.BuildInfo())
 			}
 		},
 		ValidArgsFunction: cobra.NoFileCompletions,
 	}
-	versionCommand.Flags().BoolVarP(&versionVerbose, "verbose", "v", false, "print verbose version info")
 	rootCommand.AddCommand(versionCommand)
 
+	rootCommand.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print verbose info or enable debug logger")
+
 	return rootCommand
 }
 
@@ -129,6 +132,13 @@ func coreCmd(_ *cobra.Command, args []string) {
 }
 
 func execute(attachPid int, exeFile, coreFile, outFile string, conf *config.Config) int {
+	if verbose {
+		if err := logflags.Setup(verbose, "", ""); err != nil {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+			return 1
+		}
+		defer logflags.Close()
+	}
 	if loadConfErr != nil {
 		logflags.DebuggerLogger().Errorf("%v", loadConfErr)
 	}

pkg/proc/address.go

@@ -26,3 +26,9 @@ func (a Address) Sub(b Address) int64 {
 func (a Address) Add(x int64) Address {
 	return a + Address(x)
 }
+
+// Align rounds a up to a multiple of x.
+// x must be a power of 2.
+func (a Address) Align(x int64) Address {
+	return (a + Address(x) - 1) & ^(Address(x) - 1)
+}

pkg/proc/eval.go

@@ -167,7 +167,7 @@ func extractVarInfoFromEntry(bi *proc.BinaryInfo, image *proc.Image, regs op.Dwa
 	t, err = resolveParametricType(bi, mem, t, dictAddr, mds)
 	if err != nil {
 		// Log the error, keep going with t, which will be the shape type
-		logflags.DebuggerLogger().Errorf("could not resolve parametric type of %s: %v", n, err)
+		logflags.DebuggerLogger().Warnf("could not resolve parametric type of %s: %v", n, err)
 	}
 
 	addr, pieces, _, _ := bi.Location(entry, dwarf.AttrLocation, regs.PC(), regs, mem)

pkg/proc/gcmask.go

@@ -0,0 +1,84 @@
+// Copyright 2024 CloudWeGo Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proc
+
+import (
+	"errors"
+	"math/bits"
+)
+
+type gcMaskBitIterator struct {
+	base, end Address // cannot reach end
+
+	maskBase Address
+	mask     []uint64
+
+	addr Address // iterator address
+}
+
+// nextPtr returns next ptr address starts from 'addr', returns 0 if not found.
+// If ack == true, the 'addr' will automatically increment to the next
+// starting address to be searched.
+func (b *gcMaskBitIterator) nextPtr(ack bool) Address {
+	if b == nil {
+		return 0
+	}
+	startOffset, endOffset := b.addr.Sub(b.maskBase), b.end.Sub(b.maskBase)
+	if startOffset >= endOffset || startOffset < 0 {
+		return 0
+	}
+	for startOffset < endOffset {
+		ptrIdx := startOffset / 8 / 64
+		i := startOffset / 8 % 64
+		j := int64(bits.TrailingZeros64(b.mask[ptrIdx] >> i))
+		if j == 64 {
+			// search the next ptr
+			startOffset = (ptrIdx + 1) * 64 * 8
+			continue
+		}
+		addr := b.maskBase.Add(startOffset + j*8)
+		if addr >= b.end {
+			return 0
+		}
+		if ack {
+			b.addr = addr.Add(8)
+		}
+		return addr
+	}
+	return 0
+}
+
+// resetGCMask will reset ptrMask corresponding to the address,
+// which will never be marked again by the finalMark.
+func (b *gcMaskBitIterator) resetGCMask(addr Address) error {
+	if b == nil {
+		return nil
+	}
+	if addr < b.base || addr >= b.end {
+		return errOutOfRange
+	}
+	// TODO: check gc mask
+	offset := addr.Sub(b.maskBase)
+	b.mask[offset/8/64] &= ^(1 << (offset / 8 % 64))
+	return nil
+}
+
+func newGCBitsIterator(base, end, maskBase Address, ptrMask []uint64) *gcMaskBitIterator {
+	return &gcMaskBitIterator{base: base, end: end, mask: ptrMask, addr: base, maskBase: maskBase}
+}
+
+// To avoid traversing fields/elements that escape the actual valid scope.
+// e.g. (*[1 << 16]scase)(unsafe.Pointer(cas0)) in runtime.selectgo.
+var errOutOfRange = errors.New("out of heap span range")