support combined format .tiflow file (#178)

Signed-off-by: Liu Cong <[email protected]>

CHANGELOG.md

@@ -2,8 +2,9 @@
 
 All notable changes to this project are documented in this file.
 
-## [1.3.0] - 2022-06-15
+## [1.3.1] - 2022-06-15
 + New Features
+  + Support combined meta file (#178)
   + Support macro definition in meta file (#176)
   + Env snapshot manage toolbox `env.snapshot.*` (#171)
   + Add break-point command: `break.here`

pkg/builtin/env.go

@@ -23,8 +23,8 @@ func LoadDefaultEnv(env *core.Env) {
 	env.SetInt("sys.execute-delay-sec", 0)
 	env.SetBool("sys.interact", true)
 
-	env.Set("sys.version", "1.3.0")
-	env.Set("sys.dev.name", "macro")
+	env.Set("sys.version", "1.3.1")
+	env.Set("sys.dev.name", "subflows")
 
 	env.SetBool("sys.env.use-cmd-abbrs", false)
 

pkg/builtin/flow.go

@@ -347,7 +347,7 @@ func loadFlowsFromDir(
 		}
 		cmdPath := filepath.Base(path[0 : len(path)-len(flowExt)])
 		cmdPaths := strings.Split(cmdPath, cc.Cmds.Strs.PathSep)
-		mod_meta.RegMod(cc, path, "", false, true, cmdPaths, cc.Cmds.Strs.AbbrsSep, envPathSep, source, panicRecover)
+		mod_meta.RegMod(cc, path, "", false, true, cmdPaths, flowExt, cc.Cmds.Strs.AbbrsSep, envPathSep, source, panicRecover)
 		return nil
 	})
 	return true

pkg/builtin/mod.go

@@ -66,7 +66,7 @@ func loadLocalMods(
 			cmdPath := filepath.Base(metaPath[0 : len(metaPath)-len(flowExt)])
 			cmdPaths := strings.Split(cmdPath, cc.Cmds.Strs.PathSep)
 			mod_meta.RegMod(cc, metaPath, "", false, true, cmdPaths,
-				abbrsSep, envPathSep, source, panicRecover)
+				flowExt, abbrsSep, envPathSep, source, panicRecover)
 			return nil
 		}
 
@@ -97,7 +97,7 @@ func loadLocalMods(
 
 		cmdPaths := strings.Split(cmdPath, string(filepath.Separator))
 		mod_meta.RegMod(cc, metaPath, targetPath, isDir, false, cmdPaths,
-			abbrsSep, envPathSep, source, panicRecover)
+			flowExt, abbrsSep, envPathSep, source, panicRecover)
 		return nil
 	})
 }

pkg/proto/flow_file/flow_file.go

@@ -1,11 +1,18 @@
 package flow_file
 
 import (
+	"fmt"
+
 	"github.com/pingcap/ticat/pkg/proto/meta_file"
 )
 
 func LoadFlowFile(path string) (flow []string, help string, abbrs string) {
-	meta := meta_file.NewMetaFile(path)
+	metas := meta_file.NewMetaFile(path)
+	if len(metas) != 1 {
+		panic(fmt.Errorf("can't load content for edit from a combined flow file"))
+	}
+	meta := metas[0].Meta
+
 	section := meta.GetGlobalSection()
 	help = section.Get("help")
 	abbrs = section.Get("abbrs")

pkg/proto/hub_meta/repo.go

@@ -8,13 +8,18 @@ import (
 )
 
 func ReadRepoListFromFile(selfName string, path string) (helpStr string, addrs []string, helpStrs []string) {
-	meta, err := meta_file.NewMetaFileEx(path)
+	metas, err := meta_file.NewMetaFileEx(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return
 		}
 		panic(fmt.Errorf("[ReadRepoListFromFile] read mod meta file '%s' failed: %v", path, err))
 	}
+	if len(metas) != 1 {
+		panic(fmt.Errorf("[ReadRepoListFromFile] repo meta file '%s' should not be a combined file", path))
+	}
+	meta := metas[0].Meta
+
 	helpStr = meta.Get("help")
 	repos := meta.GetSection("repos")
 	if repos == nil {

pkg/proto/meta_file/meta_file.go

@@ -9,6 +9,11 @@ import (
 	"strings"
 )
 
+type VirtualMetaFile struct {
+	Meta        *MetaFile
+	VirtualPath string
+}
+
 type MetaFile struct {
 	sections    SectionMap
 	orderedKeys []string
@@ -17,24 +22,56 @@ type MetaFile struct {
 	kvSep       string
 }
 
-func NewMetaFile(path string) *MetaFile {
-	meta, err := NewMetaFileEx(path)
+func NewMetaFile(path string) []VirtualMetaFile {
+	metas, err := NewMetaFileEx(path)
 	if err != nil {
 		panic(fmt.Errorf("[NewMetaFile] open mod meta file '%s' failed: %v", path, err))
 	}
-	return meta
+	return metas
 }
 
-func NewMetaFileEx(path string) (meta *MetaFile, err error) {
-	meta = CreateMetaFile(path)
-	var contents []byte
-	contents, err = ioutil.ReadFile(path)
-	if err == nil {
-		meta.parse(contents)
+func NewMetaFileEx(path string) (metas []VirtualMetaFile, err error) {
+	var content []byte
+	content, err = ioutil.ReadFile(path)
+	if err != nil {
+		return
+	}
+	paths, contents := parseCombinedFile(path, content, LineSep)
+	for i, it := range contents {
+		meta := CreateMetaFile(paths[i])
+		meta.parse(it)
+		metas = append(metas, VirtualMetaFile{meta, paths[i]})
 	}
 	return
 }
 
+func parseCombinedFile(path string, data []byte, lineSep string) (paths []string, contents [][]string) {
+	currPath := path
+	currLines := []string{}
+	raw := bytes.Split(data, []byte(lineSep))
+	for _, lineBytes := range raw {
+		line := string(bytes.TrimSpace(lineBytes))
+		if strings.HasPrefix(line, CombinedFileHint) {
+			cand := strings.TrimSpace(line[len(CombinedFileHint):])
+			if strings.HasPrefix(cand, CombinedFilePrefix1) {
+				cand = strings.TrimSpace(cand[len(CombinedFilePrefix1):])
+				if strings.HasPrefix(cand, CombinedFilePrefix2) {
+					if !(currPath == path && len(currLines) == 0) {
+						paths = append(paths, currPath)
+						contents = append(contents, currLines)
+					}
+					currPath = strings.TrimSpace(cand[len(CombinedFilePrefix2):])
+					currLines = []string{}
+				}
+			}
+		}
+		currLines = append(currLines, line)
+	}
+	paths = append(paths, currPath)
+	contents = append(contents, currLines)
+	return
+}
+
 func CreateMetaFile(path string) (meta *MetaFile) {
 	meta = &MetaFile{
 		make(SectionMap),
@@ -94,7 +131,7 @@ func (self *MetaFile) GetAll() SectionMap {
 	return self.sections
 }
 
-func (self *MetaFile) parse(data []byte) {
+func (self *MetaFile) parse(lines []string) {
 	var sectionName string
 	section := NewSection()
 	global := section
@@ -137,16 +174,14 @@ func (self *MetaFile) parse(data []byte) {
 		return true
 	}
 
-	// TODO: convert to string too many times
-	lines := bytes.Split(data, []byte(self.lineSep))
 	for i := 0; i < len(lines); i++ {
 		line := lines[i]
-		line = bytes.TrimSpace(line)
+		line = strings.TrimSpace(line)
 		size := len(line)
 		if size == 0 {
 			continue
 		}
-		if tryAppendMultiLine(string(line)) {
+		if tryAppendMultiLine(line) {
 			continue
 		}
 		if line[0] == CommentPrefix {
@@ -158,11 +193,11 @@ func (self *MetaFile) parse(data []byte) {
 			!(len(line) >= 4 && line[1] == SectionBracketLeft && line[size-2] == SectionBracketRight) {
 
 			if len(line) > 2 && line[size-2] == '/' {
-				k := string(line[1 : size-2])
+				k := line[1 : size-2]
 				v := []string{}
 				for i += 1; i < len(lines); i++ {
 					line := lines[i]
-					line = bytes.TrimSpace(line)
+					line = strings.TrimSpace(line)
 					// Keep the comments and blank lines in this format
 					//if line[0] == CommentPrefix {
 					//	continue
@@ -177,32 +212,32 @@ func (self *MetaFile) parse(data []byte) {
 						len(line) > 2 && line[1] == '/' {
 						break
 					}
-					l := string(bytes.TrimSpace(line))
+					l := strings.TrimSpace(line)
 					v = append(v, l)
 				}
 				if len(v) > 0 && len(v[len(v)-1]) == 0 {
 					v = v[:len(v)-1]
 				}
 				global.SetMultiLineVal(k, v)
 			} else {
-				sectionName = string(line[1 : size-1])
+				sectionName = line[1 : size-1]
 				section = NewSection()
 				self.sections[sectionName] = section
 			}
 			continue
 		}
 
-		pos := bytes.Index(line, []byte(self.kvSep))
+		pos := strings.Index(line, self.kvSep)
 		if pos < 0 {
 			panic(fmt.Errorf("[MetaFile.parse] bad kv format: %s", line))
 		}
 
-		k := bytes.TrimSpace(line[0:pos])
-		v := bytes.TrimSpace(line[pos+len(self.kvSep):])
-		if checkMultiLineStart(string(k), string(v)) {
+		k := strings.TrimSpace(line[0:pos])
+		v := strings.TrimSpace(line[pos+len(self.kvSep):])
+		if checkMultiLineStart(k, v) {
 			continue
 		}
-		section.Set(string(k), string(v))
+		section.Set(k, v)
 	}
 }
 
@@ -325,4 +360,8 @@ const (
 	CommentPrefix       = '#'
 	SectionBracketLeft  = '['
 	SectionBracketRight = ']'
+
+	CombinedFileHint    = "###"
+	CombinedFilePrefix1 = "file"
+	CombinedFilePrefix2 = ":"
 )

pkg/proto/mod_meta/mod_reg.go

@@ -13,11 +13,16 @@ import (
 
 func RegMod(
 	cc *core.Cli,
+	// Meta file full path
 	metaPath string,
+	// Related executable file full path
 	executablePath string,
 	isDir bool,
+	// Has flow ext
 	isFlow bool,
+	// Cmd path defined by meta file base name
 	cmdPath []string,
+	flowExt string,
 	abbrsSep string,
 	envPathSep string,
 	source string,
@@ -32,9 +37,50 @@ func RegMod(
 		}
 	}()
 
-	mod := cc.Cmds.GetOrAddSubEx(source, cmdPath...)
-	meta := meta_file.NewMetaFile(metaPath)
+	metas := meta_file.NewMetaFile(metaPath)
+	if len(metas) == 0 {
+		panic(fmt.Errorf("[RegMod] should never happen: NewMetaFile return no meta"))
+	}
+
+	if len(metas) == 1 {
+		regModFile(metas[0].Meta, cc, metaPath, executablePath, isDir, isFlow, cmdPath, abbrsSep, envPathSep, source, panicRecover)
+		return
+	}
+
+	if len(executablePath) != 0 || isDir || !isFlow {
+		panic(fmt.Errorf("[RegMod] should never happen: a combined flow file not meet the expection"))
+	}
+
+	// Discard the cmdPath from real file name
+	for _, meta := range metas {
+		path := getVirtualFileCmdPath(meta.VirtualPath, flowExt, cc.Cmds.Strs.PathSep)
+		regModFile(meta.Meta, cc, metaPath, "", false, true, path, abbrsSep, envPathSep, source, panicRecover)
+	}
+}
+
+func getVirtualFileCmdPath(path string, flowExt string, pathSep string) []string {
+	base := filepath.Base(path)
+	if !strings.HasSuffix(base, flowExt) {
+		panic(fmt.Errorf("virtual flow file '%s' ext not match '%s'", path, flowExt))
+	}
+	raw := base[:len(base)-len(flowExt)]
+	return strings.Split(raw, pathSep)
+}
 
+func regModFile(
+	meta *meta_file.MetaFile,
+	cc *core.Cli,
+	metaPath string,
+	executablePath string,
+	isDir bool,
+	isFlow bool,
+	cmdPath []string,
+	abbrsSep string,
+	envPathSep string,
+	source string,
+	panicRecover bool) {
+
+	mod := cc.Cmds.GetOrAddSubEx(source, cmdPath...)
 	mod.SetSource(source)
 	cmd := regMod(meta, mod, executablePath, isDir)
 	cmd.SetMetaFile(metaPath)