Merge pull request #8 from anc95/feat-comments-support

feat: add comments-support

src/ast/__snapshots__/ast_test.snap

@@ -3,20 +3,105 @@
 ast.File{
     Name: "some",
     Body: {
-        ast.TypeDeclaration{Id:"Status", Kind:"int"},
+        ast.TypeDeclaration{
+            BaseDeclaration: ast.BaseDeclaration{
+                LeadingComments: {
+                    {Value:" hello\n"},
+                },
+                TrailingComments: {
+                },
+            },
+            Id:   "Status",
+            Kind: "int",
+        },
         ast.ConstDeclaration{
-            Declarators: {
-                {Kind:"Status", Id:"Todo", Value:"iota"},
-                {Kind:"", Id:"Done", Value:""},
-                {Kind:"", Id:"Pending", Value:""},
-                {Kind:"", Id:"InProgress", Value:""},
+            BaseDeclaration: ast.BaseDeclaration{},
+            Declarators:     {
+                {
+                    BaseDeclaration: ast.BaseDeclaration{
+                        LeadingComments: {
+                            {Value:" 代办\n"},
+                        },
+                        TrailingComments: {
+                            {Value:" 59todo\n"},
+                        },
+                    },
+                    Kind:  "Status",
+                    Id:    "Todo",
+                    Value: "iota",
+                },
+                {
+                    BaseDeclaration: ast.BaseDeclaration{
+                        LeadingComments: {
+                            {Value:" 已完成\n"},
+                        },
+                        TrailingComments: {
+                        },
+                    },
+                    Kind:  "",
+                    Id:    "Done",
+                    Value: "",
+                },
+                {
+                    BaseDeclaration: ast.BaseDeclaration{
+                        LeadingComments: {
+                        },
+                        TrailingComments: {
+                        },
+                    },
+                    Kind:  "",
+                    Id:    "Pending",
+                    Value: "",
+                },
+                {
+                    BaseDeclaration: ast.BaseDeclaration{
+                        LeadingComments: {
+                        },
+                        TrailingComments: {
+                        },
+                    },
+                    Kind:  "",
+                    Id:    "InProgress",
+                    Value: "",
+                },
+            },
+        },
+        ast.TypeDeclaration{
+            BaseDeclaration: ast.BaseDeclaration{
+                LeadingComments: {
+                },
+                TrailingComments: {
+                },
             },
+            Id:   "Sex",
+            Kind: "string",
         },
-        ast.TypeDeclaration{Id:"Sex", Kind:"string"},
         ast.ConstDeclaration{
-            Declarators: {
-                {Kind:"Sex", Id:"Female", Value:"female"},
-                {Kind:"Sex", Id:"Male", Value:"male"},
+            BaseDeclaration: ast.BaseDeclaration{},
+            Declarators:     {
+                {
+                    BaseDeclaration: ast.BaseDeclaration{
+                        LeadingComments: {
+                        },
+                        TrailingComments: {
+                        },
+                    },
+                    Kind:  "Sex",
+                    Id:    "Female",
+                    Value: "female",
+                },
+                {
+                    BaseDeclaration: ast.BaseDeclaration{
+                        LeadingComments: {
+                        },
+                        TrailingComments: {
+                            {Value:"hhh\n"},
+                        },
+                    },
+                    Kind:  "Sex",
+                    Id:    "Male",
+                    Value: "male",
+                },
             },
         },
     },

src/ast/ast.go

@@ -11,20 +11,29 @@ type AstGenerator struct {
 	Tokens       []token.Token
 	index        int
 	currentToken token.Token
+	usedComments map[token.Token]bool
 }
 
 func (a *AstGenerator) nextToken(reportErrorWhenIsNull bool) (token.Token, error) {
-	a.index += 1
+	for i := a.index + 1; ; i++ {
+		if i >= len(a.Tokens) {
+			if reportErrorWhenIsNull {
+				a.reportTokenError()
+			}
 
-	if a.index >= len(a.Tokens) {
-		if reportErrorWhenIsNull {
-			a.reportTokenError()
+			return token.Token{}, errors.New("Overflow")
 		}
 
-		return token.Token{}, errors.New("Overflow")
-	}
+		tok := a.Tokens[i]
 
-	a.currentToken = a.Tokens[a.index]
+		if tok.Type == token.LineComment {
+			continue
+		}
+
+		a.index = i
+		a.currentToken = a.Tokens[a.index]
+		break
+	}
 
 	return a.currentToken, nil
 }
@@ -75,9 +84,51 @@ func (a *AstGenerator) initFile() File {
 	return file
 }
 
+func (a *AstGenerator) resolveComments(node *BaseDeclaration, leading bool) {
+	currentToken := a.currentToken
+	comments := []Comment{}
+
+	if leading {
+		for i := a.index - 1; i >= 0; i-- {
+			tok := a.Tokens[i]
+
+			if tok.Type == token.LineComment && !a.usedComments[tok] {
+				comment := Comment{}
+				comment.Value = tok.Value
+				comments = append([]Comment{comment}, comments...)
+				a.usedComments[tok] = true
+			} else {
+				break
+			}
+		}
+
+		node.LeadingComments = comments
+	} else {
+		for i := a.index + 1; i < len(a.Tokens); i++ {
+			tok := a.Tokens[i]
+
+			if tok.Start[0] != currentToken.Start[0] {
+				break
+			}
+
+			if tok.Type == token.LineComment {
+				comments = []Comment{{Value: tok.Value}}
+				a.usedComments[tok] = true
+				break
+			}
+
+			break
+		}
+
+		node.TrailingComments = comments
+	}
+}
+
 func (a *AstGenerator) readTypeDeclaration() TypeDeclaration {
 	d := TypeDeclaration{}
 
+	a.resolveComments(&d.BaseDeclaration, true)
+
 	next, _ := a.nextToken(true)
 	d.Id = next.Value
 
@@ -89,6 +140,8 @@ func (a *AstGenerator) readTypeDeclaration() TypeDeclaration {
 		d.Kind = String
 	}
 
+	a.resolveComments(&d.BaseDeclaration, false)
+
 	return d
 }
 
@@ -107,6 +160,7 @@ func (a *AstGenerator) readConstDeclaration() ConstDeclaration {
 			break
 		}
 
+		a.resolveComments(&decl.BaseDeclaration, true)
 		a.match(token.Identifier)
 		prev := a.currentToken
 		decl.Id = a.currentToken.Value
@@ -125,11 +179,13 @@ func (a *AstGenerator) readConstDeclaration() ConstDeclaration {
 
 			if a.currentToken.Type == token.StringValue || a.currentToken.Type == token.IntValue || a.currentToken.Type == token.IOTA {
 				decl.Value = a.currentToken.Value
+				a.resolveComments(&decl.BaseDeclaration, false)
 				declarators = append(declarators, decl)
 			} else {
 				a.reportTokenError()
 			}
 		} else {
+			a.resolveComments(&decl.BaseDeclaration, false)
 			declarators = append(declarators, decl)
 			if a.currentToken.Type == token.RightParentheses {
 				break
@@ -141,7 +197,7 @@ func (a *AstGenerator) readConstDeclaration() ConstDeclaration {
 		a.matchNextLine()
 	}
 
-	return ConstDeclaration{declarators}
+	return ConstDeclaration{Declarators: declarators}
 }
 
 func (a *AstGenerator) match(t token.TokenType) {
@@ -153,6 +209,10 @@ func (a *AstGenerator) match(t token.TokenType) {
 func (a *AstGenerator) matchNextLine() {
 	next := a.Tokens[a.index+1]
 
+	if next.Type == token.LineComment {
+		next = a.Tokens[a.index+2]
+	}
+
 	if next.Type == token.Semicolon {
 		a.nextToken(true)
 		return
@@ -192,5 +252,5 @@ func (a *AstGenerator) Gen() File {
 }
 
 func NewAstGenerator(tokens []token.Token) AstGenerator {
-	return AstGenerator{Tokens: tokens, index: -1}
+	return AstGenerator{Tokens: tokens, index: -1, usedComments: map[token.Token]bool{}}
 }

src/ast/ast_test.go

@@ -15,7 +15,7 @@ func TestNormal(t *testing.T) {
 	a := path.Join(wd, "../test-cases/normal.go")
 	parser := token.NewParser(a)
 	tokens := parser.Parse()
-	ast := AstGenerator{Tokens: tokens, index: -1}
+	ast := NewAstGenerator(tokens)
 
 	result := ast.Gen()
 

src/ast/type.go

@@ -7,16 +7,28 @@ const (
 	Int             = "int"
 )
 
+type BaseDeclaration struct {
+	LeadingComments  []Comment
+	TrailingComments []Comment
+}
+
 type TypeDeclaration struct {
+	BaseDeclaration
 	Id   string
 	Kind TypeKind
 }
 
 type ConstDeclaration struct {
+	BaseDeclaration
 	Declarators []ConstDeclarator
 }
 
+type Comment struct {
+	Value string
+}
+
 type ConstDeclarator struct {
+	BaseDeclaration
 	Kind  string
 	Id    string
 	Value string

src/generator/__snapshots__/generator_test.snap

@@ -3,13 +3,17 @@
 namespace some {
   export enum Sex {
     Female = 'female',
-    Male = 'male',
+    Male = 'male', //hhh
+
   }
   export enum Status {
-    Todo = 0,
+    // 已完成
     Done = 1,
-    Pending = 2,
     InProgress = 3,
+    Pending = 2,
+    // 代办
+    Todo = 0, // 59todo
+
   }
 }