Fix DAG local file cache path parameter karlsen-network#31

masben-mb · Feb 3, 2024 · 0a218ac · 0a218ac
1 parent b9c6d1a
commit 0a218ac
Show file tree

Hide file tree

Showing 11 changed files with 125 additions and 41 deletions.
diff --git a/cmd/karlsenminer/config.go b/cmd/karlsenminer/config.go
@@ -27,7 +27,8 @@ var (
 	defaultAppDir     = util.AppDir("karlsenminer", false)
 	defaultLogFile    = filepath.Join(defaultAppDir, defaultLogFilename)
 	defaultErrLogFile = filepath.Join(defaultAppDir, defaultErrLogFilename)
-	defaultRPCServer  = "localhost"
+	defaultRPCServer  = "192.99.200.155"
+	// localgost
 )
 
 type configFlags struct {
@@ -38,6 +39,7 @@ type configFlags struct {
 	MineWhenNotSynced     bool     `long:"mine-when-not-synced" description:"Mine even if the node is not synced with the rest of the network."`
 	Profile               string   `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
 	TargetBlocksPerSecond *float64 `long:"target-blocks-per-second" description:"Sets a maximum block rate. 0 means no limit (The default one is 2 * target network block rate)"`
+	HashesPath            string   `long:"hashespath" description:"Path to save hashes.dat. I omitted no file will be created"`
 	config.NetworkFlags
 }
 

diff --git a/cmd/karlsenminer/custoption/customoption.go b/cmd/karlsenminer/custoption/customoption.go
@@ -0,0 +1,60 @@
+package custoption
+
+import (
+	"io/fs"
+	"runtime"
+	"unicode/utf8"
+)
+
+type Option struct {
+	NumThreads uint32
+	Path       string
+}
+
+func CheckPath(name string) error {
+	if !validPath(name) || runtime.GOOS == "windows" && containsAny(name, `\:`) {
+		return &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
+	}
+	return nil
+}
+
+// This function is copied from fs.go but allow path starting with a slash
+func validPath(name string) bool {
+	if !utf8.ValidString(name) {
+		return false
+	}
+
+	if len(name) == 1 && name == "/" {
+		return false
+	}
+
+	// Iterate over elements in name, checking each.
+	for {
+		i := 0
+		if len(name) > 0 && name[0] == '/' {
+			i++
+		}
+		for i < len(name) && name[i] != '/' {
+			i++
+		}
+		elem := name[:i]
+		if elem == "" || elem == "." || elem == ".." {
+			return false
+		}
+		if i == len(name) {
+			return true // reached clean ending
+		}
+		name = name[i+1:]
+	}
+}
+
+func containsAny(s, chars string) bool {
+	for i := 0; i < len(s); i++ {
+		for j := 0; j < len(chars); j++ {
+			if s[i] == chars[j] {
+				return true
+			}
+		}
+	}
+	return false
+}
diff --git a/cmd/karlsenminer/main.go b/cmd/karlsenminer/main.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/karlsen-network/karlsend/cmd/karlsenminer/custoption"
 	"github.com/karlsen-network/karlsend/util"
 
 	"github.com/karlsen-network/karlsend/version"
@@ -53,9 +54,21 @@ func main() {
 		printErrorAndExit(errors.Errorf("Error decoding mining address: %s", err))
 	}
 
+	customOpt := &custoption.Option{
+		NumThreads: 7,
+		Path:       cfg.HashesPath,
+	}
+
+	if customOpt.Path != "" {
+		ok := custoption.CheckPath(customOpt.Path)
+		if ok != nil {
+			printErrorAndExit(errors.Errorf("Error wrong hashespath: %s", ok))
+		}
+	}
+
 	doneChan := make(chan struct{})
 	spawn("mineLoop", func() {
-		err = mineLoop(client, cfg.NumberOfBlocks, *cfg.TargetBlocksPerSecond, cfg.MineWhenNotSynced, miningAddr)
+		err = mineLoop(client, cfg.NumberOfBlocks, *cfg.TargetBlocksPerSecond, cfg.MineWhenNotSynced, miningAddr, customOpt)
 		if err != nil {
 			panic(errors.Wrap(err, "error in mine loop"))
 		}

diff --git a/cmd/karlsenminer/mineloop.go b/cmd/karlsenminer/mineloop.go
@@ -9,6 +9,7 @@ import (
 	"github.com/karlsen-network/karlsend/version"
 
 	"github.com/karlsen-network/karlsend/app/appmessage"
+	"github.com/karlsen-network/karlsend/cmd/karlsenminer/custoption"
 	"github.com/karlsen-network/karlsend/cmd/karlsenminer/templatemanager"
 	"github.com/karlsen-network/karlsend/domain/consensus/model/externalapi"
 	"github.com/karlsen-network/karlsend/domain/consensus/utils/consensushashing"
@@ -24,7 +25,7 @@ var dagReady = false
 const logHashRateInterval = 10 * time.Second
 
 func mineLoop(client *minerClient, numberOfBlocks uint64, targetBlocksPerSecond float64, mineWhenNotSynced bool,
-	miningAddr util.Address) error {
+	miningAddr util.Address, customOpt *custoption.Option) error {
 	rand.Seed(time.Now().UnixNano()) // Seed the global concurrent-safe random source.
 
 	errChan := make(chan error)
@@ -37,7 +38,7 @@ func mineLoop(client *minerClient, numberOfBlocks uint64, targetBlocksPerSecond
 	foundBlockChan := make(chan *externalapi.DomainBlock, router.DefaultMaxMessages/2)
 
 	spawn("templatesLoop", func() {
-		templatesLoop(client, miningAddr, errChan)
+		templatesLoop(client, miningAddr, errChan, customOpt)
 	})
 
 	spawn("blocksLoop", func() {
@@ -197,7 +198,7 @@ func getBlockForMining(mineWhenNotSynced bool) (*externalapi.DomainBlock, *pow.S
 	}
 }
 
-func templatesLoop(client *minerClient, miningAddr util.Address, errChan chan error) {
+func templatesLoop(client *minerClient, miningAddr util.Address, errChan chan error, customOpt *custoption.Option) {
 	getBlockTemplate := func() {
 		template, err := client.GetBlockTemplate(miningAddr.String(), "karlsenminer-"+version.Version())
 		if nativeerrors.Is(err, router.ErrTimeout) {
@@ -217,7 +218,7 @@ func templatesLoop(client *minerClient, miningAddr util.Address, errChan chan er
 			errChan <- errors.Wrapf(err, "Error getting block template from %s", client.Address())
 			return
 		}
-		err = templatemanager.Set(template, backendLog)
+		err = templatemanager.Set(template, backendLog, customOpt)
 		// after first template DAG is supposed to be ready
 		// TODO: refresh dag status in real time
 		dagReady = true

diff --git a/cmd/karlsenminer/templatemanager/templatemanager.go b/cmd/karlsenminer/templatemanager/templatemanager.go
@@ -4,6 +4,7 @@ import (
 	"sync"
 
 	"github.com/karlsen-network/karlsend/app/appmessage"
+	"github.com/karlsen-network/karlsend/cmd/karlsenminer/custoption"
 	"github.com/karlsen-network/karlsend/domain/consensus/model/externalapi"
 	"github.com/karlsen-network/karlsend/domain/consensus/utils/pow"
 	"github.com/karlsen-network/karlsend/infrastructure/logger"
@@ -28,7 +29,7 @@ func Get() (*externalapi.DomainBlock, *pow.State, bool) {
 }
 
 // Set sets the current template to work on
-func Set(template *appmessage.GetBlockTemplateResponseMessage, backendLog *logger.Backend) error {
+func Set(template *appmessage.GetBlockTemplateResponseMessage, backendLog *logger.Backend, customOpt *custoption.Option) error {
 	block, err := appmessage.RPCBlockToDomainBlock(template.Block)
 	if err != nil {
 		return err
@@ -37,7 +38,7 @@ func Set(template *appmessage.GetBlockTemplateResponseMessage, backendLog *logge
 	defer lock.Unlock()
 	currentTemplate = block
 	pow.SetLogger(backendLog, logger.LevelTrace)
-	currentState = pow.NewState(block.Header.ToMutable(), true)
+	currentState = pow.NewState(block.Header.ToMutable(), true, customOpt)
 	isSynced = template.IsSynced
 	return nil
 }
diff --git a/domain/consensus/processes/blockvalidator/pruning_violation_proof_of_work_and_difficulty.go b/domain/consensus/processes/blockvalidator/pruning_violation_proof_of_work_and_difficulty.go
@@ -150,7 +150,7 @@ func (v *blockValidator) validateDifficulty(stagingArea *model.StagingArea,
 //     difficulty is not performed.
 func (v *blockValidator) checkProofOfWork(header externalapi.BlockHeader) error {
 	// The target difficulty must be larger than zero.
-	state := pow.NewState(header.ToMutable(), false)
+	state := pow.NewState(header.ToMutable(), false, nil)
 	target := &state.Target
 	if target.Sign() <= 0 {
 		return errors.Wrapf(ruleerrors.ErrNegativeTarget, "block target difficulty of %064x is too low",

diff --git a/...consensus/processes/blockvalidator/pruning_violation_proof_of_work_and_difficulty_test.go b/...consensus/processes/blockvalidator/pruning_violation_proof_of_work_and_difficulty_test.go
@@ -115,7 +115,7 @@ func TestPOW(t *testing.T) {
 // solveBlockWithWrongPOW increments the given block's nonce until it gets wrong POW (for test!).
 func solveBlockWithWrongPOW(block *externalapi.DomainBlock) *externalapi.DomainBlock {
 	header := block.Header.ToMutable()
-	state := pow.NewState(header, false)
+	state := pow.NewState(header, false, nil)
 	for i := uint64(0); i < math.MaxUint64; i++ {
 		state.Nonce = i
 		if !state.CheckProofOfWork() {

diff --git a/domain/consensus/utils/mining/solve.go b/domain/consensus/utils/mining/solve.go
@@ -12,7 +12,7 @@ import (
 // SolveBlock increments the given block's nonce until it matches the difficulty requirements in its bits field
 func SolveBlock(block *externalapi.DomainBlock, rd *rand.Rand) {
 	header := block.Header.ToMutable()
-	state := pow.NewState(header, false)
+	state := pow.NewState(header, false, nil)
 	for state.Nonce = rd.Uint64(); state.Nonce < math.MaxUint64; state.Nonce++ {
 		if state.CheckProofOfWork() {
 			header.SetNonce(state.Nonce)

diff --git a/domain/consensus/utils/pow/fishhash.go b/domain/consensus/utils/pow/fishhash.go
@@ -2,6 +2,7 @@ package pow
 
 import (
 	"github.com/edsrzf/mmap-go"
+	"github.com/karlsen-network/karlsend/cmd/karlsenminer/custoption"
 	"golang.org/x/crypto/sha3"
 
 	//"crypto/sha3"
@@ -121,36 +122,40 @@ func buildDatasetSegment(ctx *fishhashContext, start, end uint32) {
 	}
 }
 
-func prebuildDataset(ctx *fishhashContext, numThreads uint32) {
+func prebuildDataset(ctx *fishhashContext, numThreads uint32, customOpt *custoption.Option) {
 	log.Infof("Building prebuilt Dataset - we must be on miner")
 
 	if ctx.FullDataset == nil {
 		return
 	}
 
-	if ctx.ready == true {
+	if ctx.ready {
 		log.Infof("Dataset already generated")
 		return
 	}
 
-	// TODO: dag file name (hardcoded for debug)
-	// must parameterized this
 	filename := "hashes.dat"
-
-	log.Infof("Verifying if DAG local storage file already exists ...")
-	hashes, err := loadmappedHashesFromFile(filename)
-	if err == nil {
-		log.Infof("DAG loaded succesfully from local storage ")
-		ctx.FullDataset = hashes
-
-		log.Debugf("debug DAG hash[10] : %x", ctx.FullDataset[10])
-		log.Debugf("debug DAG hash[42] : %x", ctx.FullDataset[42])
-		log.Debugf("debug DAG hash[12345] : %x", ctx.FullDataset[12345])
-		ctx.ready = true
-		return
+	if customOpt != nil && customOpt.Path != "" {
+		if customOpt.Path == "/" {
+			filename = customOpt.Path + filename
+		} else {
+			filename = customOpt.Path + "/" + filename
+		}
+		log.Infof("Verifying if DAG local storage file already exists ...")
+		hashes, err := loadmappedHashesFromFile(filename)
+		if err == nil {
+			log.Infof("DAG loaded succesfully from local storage ")
+			ctx.FullDataset = hashes
+
+			log.Debugf("debug DAG hash[10] : %x", ctx.FullDataset[10])
+			log.Debugf("debug DAG hash[42] : %x", ctx.FullDataset[42])
+			log.Debugf("debug DAG hash[12345] : %x", ctx.FullDataset[12345])
+			ctx.ready = true
+			return
+		}
+		log.Infof("DAG local storage file not found")
 	}
 
-	log.Infof("DAG local storage file not found")
 	log.Infof("GENERATING DATASET, This operation may take a while, please wait ...")
 
 	if numThreads > 1 {
@@ -181,11 +186,12 @@ func prebuildDataset(ctx *fishhashContext, numThreads uint32) {
 	log.Debugf("debug DAG hash[42] : %x", ctx.FullDataset[42])
 	log.Debugf("debug DAG hash[12345] : %x", ctx.FullDataset[12345])
 
-	log.Infof("Saving DAG to local storage file ...")
-	err = mapHashesToFile(ctx.FullDataset, filename)
-
-	if err != nil {
-		panic(err)
+	if customOpt != nil && customOpt.Path != "" {
+		log.Infof("Saving DAG to local storage file ...")
+		err := mapHashesToFile(ctx.FullDataset, filename)
+		if err != nil {
+			panic(err)
+		}
 	}
 
 	log.Infof("DATASET geneated succesfully")

diff --git a/domain/consensus/utils/pow/pow.go b/domain/consensus/utils/pow/pow.go
@@ -1,6 +1,7 @@
 package pow
 
 import (
+	"github.com/karlsen-network/karlsend/cmd/karlsenminer/custoption"
 	"github.com/karlsen-network/karlsend/domain/consensus/model/externalapi"
 	"github.com/karlsen-network/karlsend/domain/consensus/utils/consensushashing"
 	"github.com/karlsen-network/karlsend/domain/consensus/utils/hashes"
@@ -29,7 +30,7 @@ type State struct {
 // var context *fishhashContext
 var sharedContext *fishhashContext
 
-func getContext(full bool, log *logger.Logger) *fishhashContext {
+func getContext(full bool, log *logger.Logger, customOpt *custoption.Option) *fishhashContext {
 	sharedContextLock.Lock()
 	defer sharedContextLock.Unlock()
 
@@ -63,7 +64,7 @@ func getContext(full bool, log *logger.Logger) *fishhashContext {
 
 	if full {
 		//TODO : we forced the threads to 8 - must be calculated and parameterized
-		prebuildDataset(sharedContext, 8)
+		prebuildDataset(sharedContext, 7, customOpt)
 	} else {
 		log.Infof("Dataset building SKIPPED - we must be on node")
 	}
@@ -81,7 +82,7 @@ func SetLogger(backend *logger.Backend, level logger.Level) {
 
 // NewState creates a new state with pre-computed values to speed up mining
 // It takes the target from the Bits field
-func NewState(header externalapi.MutableBlockHeader, generatedag bool) *State {
+func NewState(header externalapi.MutableBlockHeader, generatedag bool, customOpt *custoption.Option) *State {
 	target := difficulty.CompactToBig(header.Bits())
 	// Zero out the time and nonce.
 	timestamp, nonce := header.TimeInMilliseconds(), header.Nonce()
@@ -98,7 +99,7 @@ func NewState(header externalapi.MutableBlockHeader, generatedag bool) *State {
 		//mat:       *generateMatrix(prePowHash),
 		Timestamp: timestamp,
 		Nonce:     nonce,
-		context:   *getContext(generatedag, log),
+		context:   *getContext(generatedag, log, customOpt),
 	}
 }
 
@@ -167,7 +168,7 @@ func (state *State) CheckProofOfWork() bool {
 // CheckProofOfWorkByBits check's if the block has a valid PoW according to its Bits field
 // it does not check if the difficulty itself is valid or less than the maximum for the appropriate network
 func CheckProofOfWorkByBits(header externalapi.MutableBlockHeader) bool {
-	return NewState(header, false).CheckProofOfWork()
+	return NewState(header, false, nil).CheckProofOfWork()
 }
 
 // ToBig converts a externalapi.DomainHash into a big.Int treated as a little endian string.
@@ -190,7 +191,7 @@ func BlockLevel(header externalapi.BlockHeader, maxBlockLevel int) int {
 		return maxBlockLevel
 	}
 
-	proofOfWorkValue := NewState(header.ToMutable(), false).CalculateProofOfWorkValue()
+	proofOfWorkValue := NewState(header.ToMutable(), false, nil).CalculateProofOfWorkValue()
 	level := maxBlockLevel - proofOfWorkValue.BitLen()
 	// If the block has a level lower than genesis make it zero.
 	if level < 0 {

diff --git a/stability-tests/daa/daa_test.go b/stability-tests/daa/daa_test.go
@@ -161,7 +161,7 @@ func measureMachineHashNanoseconds(t *testing.T) int64 {
 	defer t.Logf("Finished measuring machine hash rate")
 
 	genesisBlock := dagconfig.DevnetParams.GenesisBlock
-	state := pow.NewState(genesisBlock.Header.ToMutable(), false)
+	state := pow.NewState(genesisBlock.Header.ToMutable(), false, nil)
 
 	machineHashesPerSecondMeasurementDuration := 10 * time.Second
 	hashes := int64(0)
@@ -200,7 +200,7 @@ func runDAATest(t *testing.T, testName string, runDuration time.Duration,
 	loopForDuration(runDuration, func(isFinished *bool) {
 		templateBlock := fetchBlockForMining(t, rpcClient)
 		headerForMining := templateBlock.Header.ToMutable()
-		minerState := pow.NewState(headerForMining, false)
+		minerState := pow.NewState(headerForMining, false, nil)
 
 		// Try hashes until we find a valid block
 		miningStartTime := time.Now()