Docfix (#569)

* Fix docs

* Fix partial.__name__

* Fix typecheching loop with throw statements

* Cleanup

* Partial __name__ fix

* Update ci.yml

---------

Co-authored-by: Ibrahim Numanagić <[email protected]>

.github/workflows/ci.yml

@@ -100,7 +100,7 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-          - macos-11
+          - macos-12
     runs-on: ${{ matrix.os }}
     name: Codon CI
     needs: [ release ]

codon/app/main.cpp

@@ -14,6 +14,7 @@
 #include "codon/compiler/compiler.h"
 #include "codon/compiler/error.h"
 #include "codon/compiler/jit.h"
+#include "codon/parser/common.h"
 #include "codon/util/common.h"
 #include "codon/util/jupyter.h"
 #include "llvm/Support/CommandLine.h"
@@ -92,11 +93,35 @@ enum Numerics { C, Python };
 } // namespace
 
 int docMode(const std::vector<const char *> &args, const std::string &argv0) {
+  llvm::cl::opt<std::string> input(llvm::cl::Positional,
+                                   llvm::cl::desc("<input directory or file>"),
+                                   llvm::cl::init("-"));
   llvm::cl::ParseCommandLineOptions(args.size(), args.data());
   std::vector<std::string> files;
-  for (std::string line; std::getline(std::cin, line);)
-    files.push_back(line);
+  auto collectPaths = [&files](const std::string &path) {
+    llvm::sys::fs::file_status status;
+    llvm::sys::fs::status(path, status);
+    if (!llvm::sys::fs::exists(status)) {
+      codon::compilationError(fmt::format("'{}' does not exist", path), "", 0, 0, 0, -1,
+                              false);
+    }
+    if (llvm::sys::fs::is_regular_file(status)) {
+      files.emplace_back(path);
+    } else if (llvm::sys::fs::is_directory(status)) {
+      std::error_code ec;
+      for (llvm::sys::fs::recursive_directory_iterator it(path, ec), e; it != e;
+           it.increment(ec)) {
+        auto status = it->status();
+        if (!status)
+          continue;
+        if (status->type() == llvm::sys::fs::file_type::regular_file)
+          if (!codon::ast::endswith(it->path(), "__init_test__.codon"))
+            files.emplace_back(it->path());
+      }
+    }
+  };
 
+  collectPaths(args[1]);
   auto compiler = std::make_unique<codon::Compiler>(args[0]);
   bool failed = false;
   auto result = compiler->docgen(files);

codon/parser/visitors/doc/doc.cpp

@@ -92,33 +92,27 @@ std::shared_ptr<json> DocVisitor::apply(const std::string &argv0,
   shared->argv0 = argv0;
   auto cache = std::make_unique<ast::Cache>(argv0);
   shared->cache = cache.get();
-
-  auto stdlib = getImportFile(argv0, "internal", "", true, "");
-  auto ast = ast::parseFile(shared->cache, stdlib->path);
   shared->modules[""] = std::make_shared<DocContext>(shared);
-  shared->modules[""]->setFilename(stdlib->path);
   shared->j = std::make_shared<json>();
-  for (auto &s : std::vector<std::string>{"byte", "float", "bool", "int", "str",
-                                          "pyobj", "Ptr", "Function", "Generator",
-                                          "Tuple", "Int", "UInt", TYPE_OPTIONAL,
-                                          "Callable", "NoneType", "__internal__"}) {
-    shared->j->set(std::to_string(shared->itemID),
-                   std::make_shared<json>(std::unordered_map<std::string, std::string>{
-                       {"kind", "class"}, {"name", s}, {"type", "type"}}));
-    if (s == "Ptr" || s == "Generator" || s == TYPE_OPTIONAL)
-      shared->generics[shared->itemID] = {"T"};
-    if (s == "Int" || s == "UInt")
-      shared->generics[shared->itemID] = {"N"};
-    shared->modules[""]->add(s, std::make_shared<int>(shared->itemID++));
-  }
 
+  auto stdlib = getImportFile(argv0, STDLIB_INTERNAL_MODULE, "", true, "");
+  auto ast = ast::parseFile(shared->cache, stdlib->path);
+  auto core =
+      ast::parseCode(shared->cache, stdlib->path, "from internal.core import *");
+  shared->modules[""]->setFilename(stdlib->path);
+  shared->modules[""]->add("__py_numerics__", std::make_shared<int>(shared->itemID++));
+  shared->modules[""]->add("__py_extension__", std::make_shared<int>(shared->itemID++));
+  shared->modules[""]->add("__debug__", std::make_shared<int>(shared->itemID++));
+  shared->modules[""]->add("__apple__", std::make_shared<int>(shared->itemID++));
+  DocVisitor(shared->modules[""]).transformModule(std::move(core));
   DocVisitor(shared->modules[""]).transformModule(std::move(ast));
-  auto ctx = std::make_shared<DocContext>(shared);
 
+  auto ctx = std::make_shared<DocContext>(shared);
   for (auto &f : files) {
     auto path = getAbsolutePath(f);
     ctx->setFilename(path);
-    ast = ast::parseFile(shared->cache, path);
+    LOG("-> parsing {}", path);
+    auto ast = ast::parseFile(shared->cache, path);
     DocVisitor(ctx).transformModule(std::move(ast));
   }
 
@@ -156,6 +150,8 @@ std::vector<StmtPtr> DocVisitor::flatten(StmtPtr stmt, std::string *docstr, bool
 }
 
 std::shared_ptr<json> DocVisitor::transform(const ExprPtr &expr) {
+  if (!expr)
+    return std::make_shared<json>();
   DocVisitor v(ctx);
   v.setSrcInfo(expr->getSrcInfo());
   v.resultExpr = std::make_shared<json>();
@@ -164,6 +160,8 @@ std::shared_ptr<json> DocVisitor::transform(const ExprPtr &expr) {
 }
 
 std::string DocVisitor::transform(const StmtPtr &stmt) {
+  if (!stmt)
+    return "";
   DocVisitor v(ctx);
   v.setSrcInfo(stmt->getSrcInfo());
   stmt->accept(v);
@@ -249,7 +247,7 @@ void DocVisitor::visit(FunctionStmt *stmt) {
       a.status = Param::Generic;
     }
   for (auto &a : stmt->args)
-    if (a.status != Param::Normal) {
+    if (a.status == Param::Normal) {
       auto j = std::make_shared<json>();
       j->set("name", a.name);
       if (a.type)
@@ -311,7 +309,7 @@ void DocVisitor::visit(ClassStmt *stmt) {
   for (auto &g : generics)
     ctx->add(g, std::make_shared<int>(0));
   for (auto &a : stmt->args)
-    if (a.status != Param::Normal) {
+    if (a.status == Param::Normal) {
       auto ja = std::make_shared<json>();
       ja->set("name", a.name);
       if (a.type)
@@ -348,7 +346,7 @@ std::shared_ptr<json> DocVisitor::jsonify(const codon::SrcInfo &s) {
 }
 
 void DocVisitor::visit(ImportStmt *stmt) {
-  if (stmt->from->isId("C") || stmt->from->isId("python")) {
+  if (stmt->from && (stmt->from->isId("C") || stmt->from->isId("python"))) {
     int id = ctx->shared->itemID++;
     std::string name, lib;
     if (auto i = stmt->what->getId())
@@ -381,19 +379,21 @@ void DocVisitor::visit(ImportStmt *stmt) {
 
   std::vector<std::string> dirs; // Path components
   Expr *e = stmt->from.get();
-  while (auto d = e->getDot()) {
-    dirs.push_back(d->member);
-    e = d->expr.get();
+  if (e) {
+    while (auto d = e->getDot()) {
+      dirs.push_back(d->member);
+      e = d->expr.get();
+    }
+    if (!e->getId() || !stmt->args.empty() || stmt->ret ||
+        (stmt->what && !stmt->what->getId()))
+      error("invalid import statement");
+    // We have an empty stmt->from in "from .. import".
+    if (!e->getId()->value.empty())
+      dirs.push_back(e->getId()->value);
   }
-  if (!e->getId() || !stmt->args.empty() || stmt->ret ||
-      (stmt->what && !stmt->what->getId()))
-    error("invalid import statement");
-  // We have an empty stmt->from in "from .. import".
-  if (!e->getId()->value.empty())
-    dirs.push_back(e->getId()->value);
   // Handle dots (e.g. .. in from ..m import x).
   seqassert(stmt->dots >= 0, "negative dots in ImportStmt");
-  for (int i = 0; i < stmt->dots - 1; i++)
+  for (size_t i = 1; i < stmt->dots; i++)
     dirs.emplace_back("..");
   std::string path;
   for (int i = int(dirs.size()) - 1; i >= 0; i--)
@@ -406,8 +406,9 @@ void DocVisitor::visit(ImportStmt *stmt) {
   auto ictx = ctx;
   auto it = ctx->shared->modules.find(file->path);
   if (it == ctx->shared->modules.end()) {
-    ictx = std::make_shared<DocContext>(ctx->shared);
+    ctx->shared->modules[file->path] = ictx = std::make_shared<DocContext>(ctx->shared);
     ictx->setFilename(file->path);
+    LOG("=> parsing {}", file->path);
     auto tmp = parseFile(ctx->shared->cache, file->path);
     DocVisitor(ictx).transformModule(std::move(tmp));
   } else {
@@ -416,6 +417,9 @@ void DocVisitor::visit(ImportStmt *stmt) {
 
   if (!stmt->what) {
     // TODO: implement this corner case
+    for (auto &i : dirs)
+      if (!ctx->find(i))
+        ctx->add(i, std::make_shared<int>(ctx->shared->itemID++));
   } else if (stmt->what->isId("*")) {
     for (auto &i : *ictx)
       ctx->add(i.first, i.second.front());

codon/parser/visitors/typecheck/access.cpp

@@ -149,6 +149,9 @@ ExprPtr TypecheckVisitor::transformDot(DotExpr *expr,
   if (expr->expr->type->getFunc() && expr->member == "__name__") {
     return transform(N<StringExpr>(expr->expr->type->prettyString()));
   }
+  if (expr->expr->type->getPartial() && expr->member == "__name__") {
+    return transform(N<StringExpr>(expr->expr->type->getPartial()->prettyString()));
+  }
   // Special case: fn.__llvm_name__ or obj.__llvm_name__
   if (expr->member == "__llvm_name__") {
     if (realize(expr->expr->type))

codon/parser/visitors/typecheck/error.cpp

@@ -139,10 +139,11 @@ void TypecheckVisitor::visit(ThrowStmt *stmt) {
 
   transform(stmt->expr);
 
-  if (!(stmt->expr->getCall() &&
-        stmt->expr->getCall()->expr->isId("__internal__.set_header:0"))) {
+  if (!(stmt->expr->getCall() && stmt->expr->getCall()->expr->getId() &&
+        startswith(stmt->expr->getCall()->expr->getId()->value,
+                   "__internal__.set_header:0"))) {
     stmt->expr = transform(N<CallExpr>(
-        N<DotExpr>(N<IdExpr>("__internal__"), "set_header"), stmt->expr,
+        N<IdExpr>("__internal__.set_header:0"), stmt->expr,
         N<StringExpr>(ctx->getRealizationBase()->name),
         N<StringExpr>(stmt->getSrcInfo().file), N<IntExpr>(stmt->getSrcInfo().line),
         N<IntExpr>(stmt->getSrcInfo().col)));

docs/docgen.py

@@ -8,55 +8,44 @@
 import collections
 from pprint import pprint
 
-from sphinxcontrib.napoleon.docstring import GoogleDocstring
-from sphinxcontrib.napoleon import Config
+from sphinx.ext.napoleon.docstring import GoogleDocstring
+from sphinx.ext.napoleon import Config
 
 napoleon_config=Config(napoleon_use_param=True,napoleon_use_rtype=True)
 
-root=os.path.abspath(sys.argv[1])
-print(f"Generating documentation for {root}...")
-
-
-# 1. Call codon -docstr and get a documentation in JSON format
-def load_json(directory):
-    # Get all codon files in the directory
-    files=[]
-    for root,_,items in os.walk(directory):
-        for f in items:
-            if f.endswith('.codon') and "__init_test__.codon" not in f:
-                files.append(os.path.abspath(os.path.join(root,f)))
-    files='\n'.join(files)
-    s=sp.run(['../../build/codon','doc'],stdout=sp.PIPE,input=files.encode('utf-8'))
-    if s.returncode!=0:
-        raise ValueError('codon failed')
-    return json.loads(s.stdout.decode('utf-8'))
-
-
-j=load_json(root)
-print(f" - Done with codon")
-sys.exit(0)
+json_path=os.path.abspath(sys.argv[1])
+out_path=os.path.abspath(sys.argv[2])
+roots=sys.argv[3:]
+print(f"Generating documentation for {json_path}...")
+with open(json_path) as f:
+    j=json.load(f)
+print(f"Load done!")
+# sys.exit(0)
 # with open('x.json','w') as f:
 #     json.dump(j,f,indent=2)
 
 # 2. Get the list of modules and create the documentation tree
 modules={k:v["path"] for k,v in j.items() if v["kind"]=="module"}
 prefix=os.path.commonprefix(list(modules.values()))
 parsed_modules=collections.defaultdict(set)
-os.system("rm -rf stdlib/*")
+# os.system("rm -rf stdlib/*")
+root=""
 for mid,module in modules.items():
-    while module!=root:
+    while module not in roots:
         directory,name=os.path.split(module)
-        print(root,mid,module)
         directory=os.path.relpath(directory,root)  # remove the prefix
-        os.makedirs(f"stdlib/{directory}",exist_ok=True)
+        os.makedirs(f"{out_path}/{directory}",exist_ok=True)
         if name.endswith('.codon'):
             name=name[:-6] # drop suffix
         if name!='__init__':
             parsed_modules[directory].add((name,mid))
+        print(root,mid,module, '->',name)
         module=os.path.split(module)[0]
+print(f"Module read done!")
+
 for directory,modules in parsed_modules.items():
     module=directory.replace('/','.')
-    with open(f'stdlib/{directory}/index.rst','w') as f:
+    with open(f'{out_path}/{directory}/index.rst','w') as f:
         if module!='.':
             print(f".. codon:module:: {module}\n",file=f)
             print(f"{module}",file=f)
@@ -92,11 +81,15 @@ def parse_docstr(s,level=1):
 
 def parse_type(a):
     """Parse type signature"""
+    if not a:
+        return ''
     s=''
     if isinstance(a,list):
         head,tail=a[0],a[1:]
     else:
         head,tail=a,[]
+    if head not in j:
+        return '?'
     s+=j[head]["name"] if head[0].isdigit() else head
     if tail:
         for ti,t in enumerate(tail):
@@ -121,6 +114,7 @@ def parse_fn(v,skip_self=False,skip_braces=False):
         cnt+=1
         s+=f'{a["name"]}'
         if "type" in a:
+            print(a)
             s+=" : "+parse_type(a["type"])
         if "default" in a:
             s+=" = "+a["default"]+""
@@ -138,11 +132,11 @@ def parse_fn(v,skip_self=False,skip_braces=False):
 for directory,(name,mid) in {(d,m) for d,mm in parsed_modules.items() for m in mm}:
     module=directory.replace('/','.')+f".{name}"
 
-    file,mode=f'stdlib/{directory}/{name}.rst','w'
+    file,mode=f'{out_path}/{directory}/{name}.rst','w'
     if os.path.isdir(f'{root}/{directory}/{name}'):
         continue
     if name=='__init__':
-        file,mode=f'stdlib/{directory}/index.rst','a'
+        file,mode=f'{out_path}/{directory}/index.rst','a'
     with open(file,mode) as f:
         print(f".. codon:module:: {module}\n",file=f)
         print(f":codon:mod:`{module}`",file=f)