GO-4146 new spacestore

account_stop.json

@@ -0,0 +1 @@
+{"method_name":"AccountStop","middle_time":0,"error_code":101,"description":"application is not running"}

cmd/debugtree/debugtree.go

@@ -9,16 +9,19 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 
 	"github.com/goccy/go-graphviz"
+	"github.com/gogo/protobuf/jsonpb"
 	"github.com/gogo/protobuf/proto"
 
 	"github.com/anyproto/anytype-heart/core/block/editor/state"
-	"github.com/anyproto/anytype-heart/core/debug/treearchive"
+	"github.com/anyproto/anytype-heart/core/debug/exporter"
 	"github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
 	"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
 	"github.com/anyproto/anytype-heart/util/pbtypes"
@@ -42,22 +45,20 @@ func main() {
 		return
 	}
 	fmt.Println("opening file...")
-	st := time.Now()
-	archive, err := treearchive.Open(*file)
+	var (
+		st  = time.Now()
+		ctx = context.Background()
+	)
+	res, err := exporter.ImportStorage(ctx, *file)
 	if err != nil {
-		log.Fatal("can't open debug file:", err)
+		log.Fatal("can't import the tree:", err)
 	}
-	defer archive.Close()
-	fmt.Printf("open archive done in %.1fs\n", time.Since(st).Seconds())
-
-	importer := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
-	st = time.Now()
-	err = importer.Import(*fromRoot, "")
+	objectTree, err := res.CreateReadableTree(*fromRoot, "")
 	if err != nil {
-		log.Fatal("can't import the tree", err)
+		log.Fatal("can't create readable tree:", err)
 	}
-	fmt.Printf("import tree done in %.1fs\n", time.Since(st).Seconds())
-
+	fmt.Printf("open archive done in %.1fs\n", time.Since(st).Seconds())
+	importer := exporter.NewTreeImporter(objectTree)
 	if *makeJson {
 		treeJson, err := importer.Json()
 		if err != nil {
@@ -83,10 +84,11 @@ func main() {
 		}
 		fmt.Println("Change:")
 		fmt.Println(pbtypes.Sprint(ch.Model))
-		err = importer.Import(*fromRoot, ch.Id)
+		objectTree, err = res.CreateReadableTree(*fromRoot, ch.Id)
 		if err != nil {
-			log.Fatal("can't import the tree before", ch.Id, err)
+			log.Fatal("can't create readable tree:", err)
 		}
+		importer = exporter.NewTreeImporter(objectTree)
 	}
 	ot := importer.ObjectTree()
 	di, err := ot.Debug(state.ChangeParser{})
@@ -126,12 +128,16 @@ func main() {
 
 	if *objectStore {
 		fmt.Println("fetch object store info..")
-		ls, err := archive.LocalStore()
+		f, err := os.Open(filepath.Join(res.FolderPath, "localstore.json"))
 		if err != nil {
-			fmt.Println("can't open objectStore info:", err)
-		} else {
-			fmt.Println(pbtypes.Sprint(ls))
+			log.Fatal("can't open objectStore info:", err)
+		}
+		defer f.Close()
+		info := &model.ObjectInfo{}
+		if err = jsonpb.Unmarshal(f, info); err != nil {
+			log.Fatal("can't unmarshal objectStore info:", err)
 		}
+		fmt.Println(pbtypes.Sprint(info))
 	}
 
 	if *makeTree {

cmd/grpcserver/performance.go

@@ -306,7 +306,7 @@ func getTableSizes(mw *core.Middleware) (tables map[string]uint64) {
 	tables = make(map[string]uint64)
 	cfg := mw.GetApp().MustComponent(config.CName).(*config.Config)
 
-	db, err := sql.Open("sqlite3", cfg.GetSpaceStorePath())
+	db, err := sql.Open("sqlite3", cfg.GetSqliteStorePath())
 	if err != nil {
 		fmt.Println("Error opening database:", err)
 		return

core/account.go

@@ -10,6 +10,7 @@ import (
 	"github.com/anyproto/anytype-heart/core/application"
 	"github.com/anyproto/anytype-heart/core/session"
 	"github.com/anyproto/anytype-heart/pb"
+	"github.com/anyproto/anytype-heart/space/spacecore/storage/migrator"
 )
 
 func (mw *Middleware) AccountCreate(cctx context.Context, req *pb.RpcAccountCreateRequest) *pb.RpcAccountCreateResponse {
@@ -49,6 +50,35 @@ func (mw *Middleware) AccountRecover(cctx context.Context, _ *pb.RpcAccountRecov
 	}
 }
 
+func (mw *Middleware) AccountMigrate(cctx context.Context, req *pb.RpcAccountMigrateRequest) *pb.RpcAccountMigrateResponse {
+	err := mw.applicationService.AccountMigrate(cctx, req)
+	code := mapErrorCode(err,
+		errToCode(application.ErrBadInput, pb.RpcAccountMigrateResponseError_BAD_INPUT),
+		errToCode(application.ErrAccountNotFound, pb.RpcAccountMigrateResponseError_ACCOUNT_NOT_FOUND),
+		errToCode(context.Canceled, pb.RpcAccountMigrateResponseError_CANCELED),
+		errTypeToCode(&migrator.NotEnoughFreeSpaceError{}, pb.RpcAccountMigrateResponseError_NOT_ENOUGH_FREE_SPACE),
+	)
+	return &pb.RpcAccountMigrateResponse{
+		Error: &pb.RpcAccountMigrateResponseError{
+			Code:        code,
+			Description: getErrorDescription(err),
+		},
+	}
+}
+
+func (mw *Middleware) AccountMigrateCancel(cctx context.Context, req *pb.RpcAccountMigrateCancelRequest) *pb.RpcAccountMigrateCancelResponse {
+	err := mw.applicationService.AccountMigrateCancel(cctx, req)
+	code := mapErrorCode(err,
+		errToCode(application.ErrBadInput, pb.RpcAccountMigrateCancelResponseError_BAD_INPUT),
+	)
+	return &pb.RpcAccountMigrateCancelResponse{
+		Error: &pb.RpcAccountMigrateCancelResponseError{
+			Code:        code,
+			Description: getErrorDescription(err),
+		},
+	}
+}
+
 func (mw *Middleware) AccountSelect(cctx context.Context, req *pb.RpcAccountSelectRequest) *pb.RpcAccountSelectResponse {
 	account, err := mw.applicationService.AccountSelect(cctx, req)
 	code := mapErrorCode(err,
@@ -64,6 +94,7 @@ func (mw *Middleware) AccountSelect(cctx context.Context, req *pb.RpcAccountSele
 		errToCode(application.ErrAnotherProcessIsRunning, pb.RpcAccountSelectResponseError_ANOTHER_ANYTYPE_PROCESS_IS_RUNNING),
 		errToCode(application.ErrIncompatibleVersion, pb.RpcAccountSelectResponseError_FAILED_TO_FETCH_REMOTE_NODE_HAS_INCOMPATIBLE_PROTO_VERSION),
 		errToCode(application.ErrFailedToStartApplication, pb.RpcAccountSelectResponseError_FAILED_TO_RUN_NODE),
+		errToCode(application.ErrAccountStoreIsNotMigrated, pb.RpcAccountSelectResponseError_ACCOUNT_STORE_NOT_MIGRATED),
 	)
 	return &pb.RpcAccountSelectResponse{
 		Config:  nil,

core/acl/aclservice.go

@@ -10,7 +10,6 @@ import (
 	"github.com/anyproto/any-sync/app"
 	"github.com/anyproto/any-sync/commonspace/acl/aclclient"
 	"github.com/anyproto/any-sync/commonspace/object/acl/list"
-	"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
 	"github.com/anyproto/any-sync/coordinator/coordinatorclient"
 	"github.com/anyproto/any-sync/coordinator/coordinatorproto"
 	"github.com/anyproto/any-sync/nodeconf"
@@ -392,7 +391,7 @@ func (a *aclService) ViewInvite(ctx context.Context, inviteCid cid.Cid, inviteFi
 	if len(recs) == 0 {
 		return domain.InviteView{}, fmt.Errorf("no acl records found for space: %s, %w", res.SpaceId, ErrAclRequestFailed)
 	}
-	store, err := liststorage.NewInMemoryAclListStorage(recs[0].Id, recs)
+	store, err := list.NewInMemoryStorage(recs[0].Id, recs)
 	if err != nil {
 		return domain.InviteView{}, convertedOrAclRequestError(err)
 	}

core/anytype/bootstrap.go

@@ -110,9 +110,12 @@ import (
 	"github.com/anyproto/anytype-heart/space/spacecore/clientserver"
 	"github.com/anyproto/anytype-heart/space/spacecore/credentialprovider"
 	"github.com/anyproto/anytype-heart/space/spacecore/localdiscovery"
+	"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
 	"github.com/anyproto/anytype-heart/space/spacecore/peermanager"
 	"github.com/anyproto/anytype-heart/space/spacecore/peerstore"
 	"github.com/anyproto/anytype-heart/space/spacecore/storage"
+	"github.com/anyproto/anytype-heart/space/spacecore/storage/migrator"
+	"github.com/anyproto/anytype-heart/space/spacecore/storage/migratorfinisher"
 	"github.com/anyproto/anytype-heart/space/spacecore/typeprovider"
 	"github.com/anyproto/anytype-heart/space/spacefactory"
 	"github.com/anyproto/anytype-heart/space/virtualspaceservice"
@@ -208,6 +211,18 @@ func appVersion(a *app.App, clientWithVersion string) string {
 	return clientWithVersion + "/middle:" + middleVersion + "/any-sync:" + anySyncVersion
 }
 
+func BootstrapMigration(a *app.App, components ...app.Component) {
+	for _, c := range components {
+		a.Register(c)
+	}
+	a.Register(migratorfinisher.New()).
+		Register(clientds.New()).
+		Register(oldstorage.New()).
+		Register(storage.New()).
+		Register(process.New()).
+		Register(migrator.New())
+}
+
 func Bootstrap(a *app.App, components ...app.Component) {
 	for _, c := range components {
 		a.Register(c)

core/anytype/config/config.go

@@ -43,6 +43,7 @@ const (
 const (
 	SpaceStoreBadgerPath = "spacestore"
 	SpaceStoreSqlitePath = "spaceStore.db"
+	SpaceStoreNewPath    = "spaceStoreNew"
 )
 
 var (
@@ -69,6 +70,7 @@ type Config struct {
 	DisableThreadsSyncEvents               bool
 	DontStartLocalNetworkSyncAutomatically bool
 	PeferYamuxTransport                    bool
+	DisableNetworkIdCheck                  bool
 	SpaceStorageMode                       storage.SpaceStorageMode
 	NetworkMode                            pb.RpcAccountNetworkMode
 	NetworkCustomConfigFilePath            string           `json:",omitempty"` // not saved to config
@@ -293,12 +295,27 @@ func (c *Config) FSConfig() (FSConfig, error) {
 	return FSConfig{IPFSStorageAddr: res.CustomFileStorePath}, nil
 }
 
+func (c *Config) GetRepoPath() string {
+	return c.RepoPath
+}
+
 func (c *Config) GetConfigPath() string {
 	return filepath.Join(c.RepoPath, ConfigFileName)
 }
 
-func (c *Config) GetSpaceStorePath() string {
-	return filepath.Join(c.RepoPath, "spaceStore.db")
+func (c *Config) GetSqliteStorePath() string {
+	return filepath.Join(c.RepoPath, SpaceStoreSqlitePath)
+}
+
+func (c *Config) GetOldSpaceStorePath() string {
+	if c.GetSpaceStorageMode() == storage.SpaceStorageModeBadger {
+		return filepath.Join(c.RepoPath, SpaceStoreBadgerPath)
+	}
+	return c.GetSqliteStorePath()
+}
+
+func (c *Config) GetNewSpaceStorePath() string {
+	return filepath.Join(c.RepoPath, SpaceStoreNewPath)
 }
 
 func (c *Config) GetTempDirPath() string {
@@ -391,7 +408,7 @@ func (c *Config) GetNodeConfWithError() (conf nodeconf.Configuration, err error)
 		if err := yaml.Unmarshal(confBytes, &conf); err != nil {
 			return nodeconf.Configuration{}, errors.Join(ErrNetworkFileFailedToRead, err)
 		}
-		if c.NetworkId != "" && c.NetworkId != conf.NetworkId {
+		if !c.DisableNetworkIdCheck && c.NetworkId != "" && c.NetworkId != conf.NetworkId {
 			log.Warnf("Network id mismatch: %s != %s", c.NetworkId, conf.NetworkId)
 			return nodeconf.Configuration{}, errors.Join(ErrNetworkIdMismatch, fmt.Errorf("network id mismatch: %s != %s", c.NetworkId, conf.NetworkId))
 		}

core/application/account_select.go

@@ -31,15 +31,23 @@ var (
 	ErrNoMnemonicProvided  = errors.New("no mnemonic provided")
 	ErrIncompatibleVersion = errors.New("can't fetch account's data because remote nodes have incompatible protocol version. Please update anytype to the latest version")
 
-	ErrAnotherProcessIsRunning = errors.New("another anytype process is running")
-	ErrFailedToFindAccountInfo = errors.New("failed to find account info")
-	ErrAccountIsDeleted        = errors.New("account is deleted")
+	ErrAnotherProcessIsRunning   = errors.New("another anytype process is running")
+	ErrFailedToFindAccountInfo   = errors.New("failed to find account info")
+	ErrAccountIsDeleted          = errors.New("account is deleted")
+	ErrAccountStoreIsNotMigrated = errors.New("account store is not migrated")
 )
 
 func (s *Service) AccountSelect(ctx context.Context, req *pb.RpcAccountSelectRequest) (*model.Account, error) {
 	if req.Id == "" {
 		return nil, ErrEmptyAccountID
 	}
+	curMigration := s.migrationManager.getOrCreateMigration(req.RootPath, req.Id)
+	if !curMigration.successful() {
+		return nil, ErrAccountStoreIsNotMigrated
+	}
+	if s.migrationManager.isRunning() {
+		return nil, ErrMigrationRunning
+	}
 
 	if runtime.GOOS != "android" && runtime.GOOS != "ios" {
 		s.traceRecorder.start()