From 189aa886198ffd855619f45f1c9666bf34b300d8 Mon Sep 17 00:00:00 2001 From: "A. R. Shajii" Date: Thu, 18 Jul 2024 08:09:30 -0400 Subject: [PATCH] Docfix (#569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix docs * Fix partial.__name__ * Fix typecheching loop with throw statements * Cleanup * Partial __name__ fix * Update ci.yml --------- Co-authored-by: Ibrahim Numanagić --- .github/workflows/ci.yml | 2 +- codon/app/main.cpp | 29 ++++++++- codon/parser/visitors/doc/doc.cpp | 70 ++++++++++++---------- codon/parser/visitors/typecheck/access.cpp | 3 + codon/parser/visitors/typecheck/error.cpp | 7 ++- docs/docgen.py | 56 ++++++++--------- 6 files changed, 97 insertions(+), 70 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ace2ce57..33292ea5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: matrix: os: - ubuntu-latest - - macos-11 + - macos-12 runs-on: ${{ matrix.os }} name: Codon CI needs: [ release ] diff --git a/codon/app/main.cpp b/codon/app/main.cpp index df487565..7d602dae 100644 --- a/codon/app/main.cpp +++ b/codon/app/main.cpp @@ -14,6 +14,7 @@ #include "codon/compiler/compiler.h" #include "codon/compiler/error.h" #include "codon/compiler/jit.h" +#include "codon/parser/common.h" #include "codon/util/common.h" #include "codon/util/jupyter.h" #include "llvm/Support/CommandLine.h" @@ -92,11 +93,35 @@ enum Numerics { C, Python }; } // namespace int docMode(const std::vector &args, const std::string &argv0) { + llvm::cl::opt input(llvm::cl::Positional, + llvm::cl::desc(""), + llvm::cl::init("-")); llvm::cl::ParseCommandLineOptions(args.size(), args.data()); std::vector files; - for (std::string line; std::getline(std::cin, line);) - files.push_back(line); + auto collectPaths = [&files](const std::string &path) { + llvm::sys::fs::file_status status; + llvm::sys::fs::status(path, status); + if (!llvm::sys::fs::exists(status)) { + codon::compilationError(fmt::format("'{}' does not exist", path), "", 0, 0, 0, -1, + false); + } + if (llvm::sys::fs::is_regular_file(status)) { + files.emplace_back(path); + } else if (llvm::sys::fs::is_directory(status)) { + std::error_code ec; + for (llvm::sys::fs::recursive_directory_iterator it(path, ec), e; it != e; + it.increment(ec)) { + auto status = it->status(); + if (!status) + continue; + if (status->type() == llvm::sys::fs::file_type::regular_file) + if (!codon::ast::endswith(it->path(), "__init_test__.codon")) + files.emplace_back(it->path()); + } + } + }; + collectPaths(args[1]); auto compiler = std::make_unique(args[0]); bool failed = false; auto result = compiler->docgen(files); diff --git a/codon/parser/visitors/doc/doc.cpp b/codon/parser/visitors/doc/doc.cpp index 9de73218..6515f74d 100644 --- a/codon/parser/visitors/doc/doc.cpp +++ b/codon/parser/visitors/doc/doc.cpp @@ -92,33 +92,27 @@ std::shared_ptr DocVisitor::apply(const std::string &argv0, shared->argv0 = argv0; auto cache = std::make_unique(argv0); shared->cache = cache.get(); - - auto stdlib = getImportFile(argv0, "internal", "", true, ""); - auto ast = ast::parseFile(shared->cache, stdlib->path); shared->modules[""] = std::make_shared(shared); - shared->modules[""]->setFilename(stdlib->path); shared->j = std::make_shared(); - for (auto &s : std::vector{"byte", "float", "bool", "int", "str", - "pyobj", "Ptr", "Function", "Generator", - "Tuple", "Int", "UInt", TYPE_OPTIONAL, - "Callable", "NoneType", "__internal__"}) { - shared->j->set(std::to_string(shared->itemID), - std::make_shared(std::unordered_map{ - {"kind", "class"}, {"name", s}, {"type", "type"}})); - if (s == "Ptr" || s == "Generator" || s == TYPE_OPTIONAL) - shared->generics[shared->itemID] = {"T"}; - if (s == "Int" || s == "UInt") - shared->generics[shared->itemID] = {"N"}; - shared->modules[""]->add(s, std::make_shared(shared->itemID++)); - } + auto stdlib = getImportFile(argv0, STDLIB_INTERNAL_MODULE, "", true, ""); + auto ast = ast::parseFile(shared->cache, stdlib->path); + auto core = + ast::parseCode(shared->cache, stdlib->path, "from internal.core import *"); + shared->modules[""]->setFilename(stdlib->path); + shared->modules[""]->add("__py_numerics__", std::make_shared(shared->itemID++)); + shared->modules[""]->add("__py_extension__", std::make_shared(shared->itemID++)); + shared->modules[""]->add("__debug__", std::make_shared(shared->itemID++)); + shared->modules[""]->add("__apple__", std::make_shared(shared->itemID++)); + DocVisitor(shared->modules[""]).transformModule(std::move(core)); DocVisitor(shared->modules[""]).transformModule(std::move(ast)); - auto ctx = std::make_shared(shared); + auto ctx = std::make_shared(shared); for (auto &f : files) { auto path = getAbsolutePath(f); ctx->setFilename(path); - ast = ast::parseFile(shared->cache, path); + LOG("-> parsing {}", path); + auto ast = ast::parseFile(shared->cache, path); DocVisitor(ctx).transformModule(std::move(ast)); } @@ -156,6 +150,8 @@ std::vector DocVisitor::flatten(StmtPtr stmt, std::string *docstr, bool } std::shared_ptr DocVisitor::transform(const ExprPtr &expr) { + if (!expr) + return std::make_shared(); DocVisitor v(ctx); v.setSrcInfo(expr->getSrcInfo()); v.resultExpr = std::make_shared(); @@ -164,6 +160,8 @@ std::shared_ptr DocVisitor::transform(const ExprPtr &expr) { } std::string DocVisitor::transform(const StmtPtr &stmt) { + if (!stmt) + return ""; DocVisitor v(ctx); v.setSrcInfo(stmt->getSrcInfo()); stmt->accept(v); @@ -249,7 +247,7 @@ void DocVisitor::visit(FunctionStmt *stmt) { a.status = Param::Generic; } for (auto &a : stmt->args) - if (a.status != Param::Normal) { + if (a.status == Param::Normal) { auto j = std::make_shared(); j->set("name", a.name); if (a.type) @@ -311,7 +309,7 @@ void DocVisitor::visit(ClassStmt *stmt) { for (auto &g : generics) ctx->add(g, std::make_shared(0)); for (auto &a : stmt->args) - if (a.status != Param::Normal) { + if (a.status == Param::Normal) { auto ja = std::make_shared(); ja->set("name", a.name); if (a.type) @@ -348,7 +346,7 @@ std::shared_ptr DocVisitor::jsonify(const codon::SrcInfo &s) { } void DocVisitor::visit(ImportStmt *stmt) { - if (stmt->from->isId("C") || stmt->from->isId("python")) { + if (stmt->from && (stmt->from->isId("C") || stmt->from->isId("python"))) { int id = ctx->shared->itemID++; std::string name, lib; if (auto i = stmt->what->getId()) @@ -381,19 +379,21 @@ void DocVisitor::visit(ImportStmt *stmt) { std::vector dirs; // Path components Expr *e = stmt->from.get(); - while (auto d = e->getDot()) { - dirs.push_back(d->member); - e = d->expr.get(); + if (e) { + while (auto d = e->getDot()) { + dirs.push_back(d->member); + e = d->expr.get(); + } + if (!e->getId() || !stmt->args.empty() || stmt->ret || + (stmt->what && !stmt->what->getId())) + error("invalid import statement"); + // We have an empty stmt->from in "from .. import". + if (!e->getId()->value.empty()) + dirs.push_back(e->getId()->value); } - if (!e->getId() || !stmt->args.empty() || stmt->ret || - (stmt->what && !stmt->what->getId())) - error("invalid import statement"); - // We have an empty stmt->from in "from .. import". - if (!e->getId()->value.empty()) - dirs.push_back(e->getId()->value); // Handle dots (e.g. .. in from ..m import x). seqassert(stmt->dots >= 0, "negative dots in ImportStmt"); - for (int i = 0; i < stmt->dots - 1; i++) + for (size_t i = 1; i < stmt->dots; i++) dirs.emplace_back(".."); std::string path; for (int i = int(dirs.size()) - 1; i >= 0; i--) @@ -406,8 +406,9 @@ void DocVisitor::visit(ImportStmt *stmt) { auto ictx = ctx; auto it = ctx->shared->modules.find(file->path); if (it == ctx->shared->modules.end()) { - ictx = std::make_shared(ctx->shared); + ctx->shared->modules[file->path] = ictx = std::make_shared(ctx->shared); ictx->setFilename(file->path); + LOG("=> parsing {}", file->path); auto tmp = parseFile(ctx->shared->cache, file->path); DocVisitor(ictx).transformModule(std::move(tmp)); } else { @@ -416,6 +417,9 @@ void DocVisitor::visit(ImportStmt *stmt) { if (!stmt->what) { // TODO: implement this corner case + for (auto &i : dirs) + if (!ctx->find(i)) + ctx->add(i, std::make_shared(ctx->shared->itemID++)); } else if (stmt->what->isId("*")) { for (auto &i : *ictx) ctx->add(i.first, i.second.front()); diff --git a/codon/parser/visitors/typecheck/access.cpp b/codon/parser/visitors/typecheck/access.cpp index ce3ae8bc..8a0e8947 100644 --- a/codon/parser/visitors/typecheck/access.cpp +++ b/codon/parser/visitors/typecheck/access.cpp @@ -149,6 +149,9 @@ ExprPtr TypecheckVisitor::transformDot(DotExpr *expr, if (expr->expr->type->getFunc() && expr->member == "__name__") { return transform(N(expr->expr->type->prettyString())); } + if (expr->expr->type->getPartial() && expr->member == "__name__") { + return transform(N(expr->expr->type->getPartial()->prettyString())); + } // Special case: fn.__llvm_name__ or obj.__llvm_name__ if (expr->member == "__llvm_name__") { if (realize(expr->expr->type)) diff --git a/codon/parser/visitors/typecheck/error.cpp b/codon/parser/visitors/typecheck/error.cpp index dd0b313a..77335233 100644 --- a/codon/parser/visitors/typecheck/error.cpp +++ b/codon/parser/visitors/typecheck/error.cpp @@ -139,10 +139,11 @@ void TypecheckVisitor::visit(ThrowStmt *stmt) { transform(stmt->expr); - if (!(stmt->expr->getCall() && - stmt->expr->getCall()->expr->isId("__internal__.set_header:0"))) { + if (!(stmt->expr->getCall() && stmt->expr->getCall()->expr->getId() && + startswith(stmt->expr->getCall()->expr->getId()->value, + "__internal__.set_header:0"))) { stmt->expr = transform(N( - N(N("__internal__"), "set_header"), stmt->expr, + N("__internal__.set_header:0"), stmt->expr, N(ctx->getRealizationBase()->name), N(stmt->getSrcInfo().file), N(stmt->getSrcInfo().line), N(stmt->getSrcInfo().col))); diff --git a/docs/docgen.py b/docs/docgen.py index 419c9d32..095e2ec8 100755 --- a/docs/docgen.py +++ b/docs/docgen.py @@ -8,33 +8,19 @@ import collections from pprint import pprint -from sphinxcontrib.napoleon.docstring import GoogleDocstring -from sphinxcontrib.napoleon import Config +from sphinx.ext.napoleon.docstring import GoogleDocstring +from sphinx.ext.napoleon import Config napoleon_config=Config(napoleon_use_param=True,napoleon_use_rtype=True) -root=os.path.abspath(sys.argv[1]) -print(f"Generating documentation for {root}...") - - -# 1. Call codon -docstr and get a documentation in JSON format -def load_json(directory): - # Get all codon files in the directory - files=[] - for root,_,items in os.walk(directory): - for f in items: - if f.endswith('.codon') and "__init_test__.codon" not in f: - files.append(os.path.abspath(os.path.join(root,f))) - files='\n'.join(files) - s=sp.run(['../../build/codon','doc'],stdout=sp.PIPE,input=files.encode('utf-8')) - if s.returncode!=0: - raise ValueError('codon failed') - return json.loads(s.stdout.decode('utf-8')) - - -j=load_json(root) -print(f" - Done with codon") -sys.exit(0) +json_path=os.path.abspath(sys.argv[1]) +out_path=os.path.abspath(sys.argv[2]) +roots=sys.argv[3:] +print(f"Generating documentation for {json_path}...") +with open(json_path) as f: + j=json.load(f) +print(f"Load done!") +# sys.exit(0) # with open('x.json','w') as f: # json.dump(j,f,indent=2) @@ -42,21 +28,24 @@ def load_json(directory): modules={k:v["path"] for k,v in j.items() if v["kind"]=="module"} prefix=os.path.commonprefix(list(modules.values())) parsed_modules=collections.defaultdict(set) -os.system("rm -rf stdlib/*") +# os.system("rm -rf stdlib/*") +root="" for mid,module in modules.items(): - while module!=root: + while module not in roots: directory,name=os.path.split(module) - print(root,mid,module) directory=os.path.relpath(directory,root) # remove the prefix - os.makedirs(f"stdlib/{directory}",exist_ok=True) + os.makedirs(f"{out_path}/{directory}",exist_ok=True) if name.endswith('.codon'): name=name[:-6] # drop suffix if name!='__init__': parsed_modules[directory].add((name,mid)) + print(root,mid,module, '->',name) module=os.path.split(module)[0] +print(f"Module read done!") + for directory,modules in parsed_modules.items(): module=directory.replace('/','.') - with open(f'stdlib/{directory}/index.rst','w') as f: + with open(f'{out_path}/{directory}/index.rst','w') as f: if module!='.': print(f".. codon:module:: {module}\n",file=f) print(f"{module}",file=f) @@ -92,11 +81,15 @@ def parse_docstr(s,level=1): def parse_type(a): """Parse type signature""" + if not a: + return '' s='' if isinstance(a,list): head,tail=a[0],a[1:] else: head,tail=a,[] + if head not in j: + return '?' s+=j[head]["name"] if head[0].isdigit() else head if tail: for ti,t in enumerate(tail): @@ -121,6 +114,7 @@ def parse_fn(v,skip_self=False,skip_braces=False): cnt+=1 s+=f'{a["name"]}' if "type" in a: + print(a) s+=" : "+parse_type(a["type"]) if "default" in a: s+=" = "+a["default"]+"" @@ -138,11 +132,11 @@ def parse_fn(v,skip_self=False,skip_braces=False): for directory,(name,mid) in {(d,m) for d,mm in parsed_modules.items() for m in mm}: module=directory.replace('/','.')+f".{name}" - file,mode=f'stdlib/{directory}/{name}.rst','w' + file,mode=f'{out_path}/{directory}/{name}.rst','w' if os.path.isdir(f'{root}/{directory}/{name}'): continue if name=='__init__': - file,mode=f'stdlib/{directory}/index.rst','a' + file,mode=f'{out_path}/{directory}/index.rst','a' with open(file,mode) as f: print(f".. codon:module:: {module}\n",file=f) print(f":codon:mod:`{module}`",file=f)