New overlays feature for doing ad-hoc simulations of existing contracts with modified bytecode

cmd/geth/consolecmd_test.go

@@ -30,7 +30,7 @@ import (
 )
 
 const (
-	ipcAPIs  = "admin:1.0 clique:1.0 debug:1.0 engine:1.0 eth:1.0 miner:1.0 net:1.0 rpc:1.0 txpool:1.0 web3:1.0"
+	ipcAPIs  = "admin:1.0 clique:1.0 debug:1.0 engine:1.0 eth:1.0 miner:1.0 net:1.0 overlay:1.0 rpc:1.0 txpool:1.0 web3:1.0"
 	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 

cmd/utils/flags.go

@@ -50,6 +50,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/eth/filters"
 	"github.com/ethereum/go-ethereum/eth/gasprice"
+	"github.com/ethereum/go-ethereum/eth/overlay"
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/remotedb"
@@ -1990,6 +1991,7 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend
 		Fatalf("Failed to register the Ethereum service: %v", err)
 	}
 	stack.RegisterAPIs(tracers.APIs(backend.APIBackend))
+	stack.RegisterAPIs(overlay.APIs(backend.APIBackend))
 	return backend.APIBackend, backend
 }
 

core/vm/evm.go

@@ -430,6 +430,14 @@ func (c *codeAndHash) Hash() common.Hash {
 
 // create creates a new contract using code as deployment code.
 func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address common.Address, typ OpCode) ([]byte, common.Address, uint64, error) {
+	if evm.Config.Tracer != nil && evm.Config.ApplyOverlayCreateHook {
+		if evm.depth == 0 {
+			evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value.ToBig())
+		} else {
+			evm.Config.Tracer.CaptureEnter(typ, caller.Address(), address, codeAndHash.code, gas, value.ToBig())
+		}
+	}
+
 	// Depth check execution. Fail if we're trying to execute above the
 	// limit.
 	if evm.depth > int(params.CallCreateDepth) {
@@ -466,7 +474,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
 	contract := NewContract(caller, AccountRef(address), value, gas)
 	contract.SetCodeOptionalHash(&address, codeAndHash)
 
-	if evm.Config.Tracer != nil {
+	if evm.Config.Tracer != nil && !evm.Config.ApplyOverlayCreateHook {
 		if evm.depth == 0 {
 			evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value.ToBig())
 		} else {
@@ -519,6 +527,14 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
 	return ret, address, contract.Gas, err
 }
 
+func NewCodeAndHash(code []byte) *codeAndHash {
+	return &codeAndHash{code: code}
+}
+
+func (evm *EVM) OverlayCreate(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address common.Address, typ OpCode, incrementNonce bool) ([]byte, common.Address, uint64, error) {
+	return evm.create(caller, codeAndHash, gas, value, address, typ)
+}
+
 // Create creates a new contract using code as deployment code.
 func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) {
 	contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))

core/vm/interpreter.go

@@ -34,6 +34,7 @@ type Config struct {
 	EnablePreimageRecording     bool                // Enables recording of SHA3/keccak preimages
 	ExtraEips                   []int               // Additional EIPS that are to be enabled
 	OptimismPrecompileOverrides PrecompileOverrides // Precompile overrides for Optimism
+	ApplyOverlayCreateHook      bool                // Enables creation of overlays
 }
 
 // ScopeContext contains the things that are per-call, such as stack and memory,

eth/backend.go

@@ -384,6 +384,9 @@ func (s *Ethereum) APIs() []rpc.API {
 		}, {
 			Namespace: "net",
 			Service:   s.netRPCService,
+		}, {
+			Namespace: "overlay",
+			Service:   ethapi.NewOverlayAPI(s.APIBackend),
 		},
 	}...)
 }

eth/overlay/README.md

@@ -0,0 +1,102 @@
+# Overlays
+Overlays allow you to add your custom logic to already deployed contracts and simulate events and calls on top of them.
+With overlays you can create new view functions, modify existing ones, change field visibility, emit new events and query the historical data of many contracts with your modified source code.
+
+## API
+This explains how to use the overlay API.
+
+### `overlay_callConstructor`
+This method needs to be called once with the new bytecode.
+
+It first does a lookup of the creationTx for the given contract.
+Once it's found, it injects the new code and returns the new creation bytecode result from the EVM to the caller.
+
+Example request:
+```json
+{
+  "id" : "1",
+  "jsonrpc" : "2.0",
+  "method" : "overlay_callConstructor",
+  "params" : [
+    "<CONTRACT_ADDRESS>",
+    "<BYTECODE>"
+  ]
+}
+```
+
+Example response:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": "1",
+  "result": {
+    "code": "<CREATION_BYTECODE>"
+  }
+}
+```
+
+### `overlay_getLogs`
+This method can be called multiple times to receive new logs from your new bytecode.
+
+It has the same interface as `eth_getLogs` but it also accepts state overrides as the second param.
+We can pass the creation bytecode from the call to `overlay_callConstructor` along to `overlay_getLogs` as state overrides.
+The passed block range for the filter defines the initial block range that needs to be replayed with the given state overrides.
+Once all blocks are replayed, the logs are returned to the caller.
+
+Example request:
+```json
+{
+   "id" : 1,
+   "jsonrpc" : "2.0",
+   "method" : "overlay_getLogs",
+   "params" : [
+      {
+         "address" : "<CONTRACT_ADDRESS>",
+         "fromBlock" : "0x6e7dd00",
+         "toBlock" : "0x6e7dd00"
+      },
+      {
+         "<CONTRACT_ADDRESS>" : {
+            "code" : "<CREATION_BYTECODE>"
+        }
+      }
+    ]
+}
+```
+
+Example response as in `eth_getLogs`:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": [
+    {
+      "address": "<CONTRACT_ADDRESS",
+      "topics": [
+        "0xaabd75b90fb7114eb9587a54f00ce5ebe8cb4a70627f3a6c26e506ffd771fe2f",
+        "0x00000000000000000000000000000000fc25870c6ed6b6c7e41fb078b7656f69",
+        "0x000000000000000000000000000000000000000000000000000000000004d77c"
+      ],
+      "data": "0x0000000000000000000000000000000000000000000000000000000000000001",
+      "blockNumber": "0x6e7dd00",
+      "transactionHash": "0x32983cb59ae25889efac5ec4850534421495b1a231c8472e822a340eff8db23e",
+      "transactionIndex": "0x1",
+      "blockHash": "0xffe6aa3ba2bcb08f8fdd340e0af309e54d2da0b7d08efb3185f6dece00d0a3c6",
+      "logIndex": "0x1",
+      "removed": false
+    }
+  ]
+}
+
+```
+
+### `eth_call`
+This method can be called multiple times to call new view functions that you defined in your new bytecode.
+
+By sending the creation bytecode received from `overlay_callConstructor` as state overrides to `eth_call` you'll be able to call new functions on your contract.
+
+## Tests
+There's a [postman collection for overlays](Overlay_Testing.json) with sample requests for `overlay_callConstructor` and `overlay_getLogs` which can be used for reference and refactorings.
+
+## Configuration
+- add `overlay` to your `--http.api` flag