From 4ebe7a2e46b60bae7c04d50980bc6a02f9402c8b Mon Sep 17 00:00:00 2001 From: Sifis Lagouvardos Date: Wed, 6 Feb 2019 19:10:06 +0200 Subject: [PATCH 1/4] Changes in imports, added new kind of instruction to correctly support from A import B instructions. --- .../python/ir/PythonCAstToIRTranslator.java | 11 ++++- .../wala/cast/python/ir/PythonLanguage.java | 18 +++++---- .../wala/cast/python/parser/PythonParser.java | 40 +++++++++++-------- .../ssa/PythonImportFromGetInstruction.java | 20 ++++++++++ .../wala/cast/python/types/PythonTypes.java | 2 + 5 files changed, 65 insertions(+), 26 deletions(-) create mode 100644 com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ssa/PythonImportFromGetInstruction.java diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java index 7e4a006e6..30b3098b4 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java @@ -229,7 +229,16 @@ public void doArrayWrite(WalkContext context, int arrayValue, CAstNode arrayRef, protected void doFieldRead(WalkContext context, int result, int receiver, CAstNode elt, CAstNode parent) { int currentInstruction = context.cfg().getCurrentInstruction(); if (elt.getKind() == CAstNode.CONSTANT && elt.getValue() instanceof String) { - FieldReference f = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom((String)elt.getValue()), PythonTypes.Root); + FieldReference f; + if(parent.getChildCount() > 1 + && parent.getChild(0).getKind() == CAstNode.PRIMITIVE + && parent.getChild(0).getChildCount() == 2 + && parent.getChild(0).getChild(0).getValue().equals("import")){ + f = FieldReference.findOrCreate(PythonTypes.module, Atom.findOrCreateUnicodeAtom((String)elt.getValue()), PythonTypes.Root); + } + else + f = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom((String)elt.getValue()), PythonTypes.Root); + context.cfg().addInstruction(Python.instructionFactory().GetInstruction(currentInstruction, result, receiver, f)); } else { visit(elt, context, this); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java index fc86a8776..2b23250b8 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java @@ -27,10 +27,7 @@ import com.ibm.wala.cast.python.cfg.PythonInducedCFG; import com.ibm.wala.cast.python.modref.PythonModRef.PythonModVisitor; import com.ibm.wala.cast.python.modref.PythonModRef.PythonRefVisitor; -import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; -import com.ibm.wala.cast.python.ssa.PythonPropertyRead; -import com.ibm.wala.cast.python.ssa.PythonPropertyWrite; -import com.ibm.wala.cast.python.ssa.PythonStoreProperty; +import com.ibm.wala.cast.python.ssa.*; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.types.AstMethodReference; import com.ibm.wala.cfg.InducedCFG; @@ -53,10 +50,7 @@ import com.ibm.wala.ipa.modref.ModRef.RefVisitor; import com.ibm.wala.shrikeCT.BootstrapMethodsReader.BootstrapMethod; import com.ibm.wala.shrikeCT.InvalidClassFileException; -import com.ibm.wala.ssa.SSAAbstractInvokeInstruction; -import com.ibm.wala.ssa.SSAArrayStoreInstruction; -import com.ibm.wala.ssa.SSAInstruction; -import com.ibm.wala.ssa.SSAInvokeInstruction; +import com.ibm.wala.ssa.*; import com.ibm.wala.types.FieldReference; import com.ibm.wala.types.MethodReference; import com.ibm.wala.types.TypeName; @@ -248,6 +242,14 @@ public EachElementGetInstruction EachElementGetInstruction(int iindex, int value public AstYieldInstruction YieldInstruction(int iindex, int[] rvals) { return new AstYieldInstruction(iindex, rvals); } + + @Override + public SSAGetInstruction GetInstruction(int iindex, int result, int ref, FieldReference field){ + if(field.getDeclaringClass().getName().toString().equals("Lmodule")) + return new PythonImportFromGetInstruction(iindex, result, ref, field); + + return super.GetInstruction(iindex, result, ref, field); + } }; } diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java index 3ea567b9f..3d6dfb032 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java @@ -1021,7 +1021,8 @@ public CAstNode visitIfExp(IfExp arg0) throws Exception { private String name(alias n) { String s = n.getInternalAsname()==null? n.getInternalName(): n.getInternalAsname(); if (s.contains(".")) { - s = s.substring(0, s.indexOf('.')); + //s = s.substring(0, s.indexOf('.')); + s = s.replace(".","/"); } return s; } @@ -1031,6 +1032,7 @@ public CAstNode visitImport(Import arg0) throws Exception { int i = 0; CAstNode[] elts = new CAstNode[ arg0.getInternalNames().size()*2 ]; for(alias n : arg0.getInternalNames()) { + System.err.println("SIMPLE IMPORT "+ n.getInternalAsname() + " | " + name(n) + " intName = " + n.getInternalName()); elts[i++] = Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any))); elts[i++] = Ast.makeNode(CAstNode.ASSIGN, @@ -1042,32 +1044,36 @@ public CAstNode visitImport(Import arg0) throws Exception { @Override public CAstNode visitImportFrom(ImportFrom arg0) throws Exception { - CAstNode[] elts = new CAstNode[ arg0.getInternalNames().size() ]; + CAstNode[] elts = new CAstNode[ arg0.getInternalNames().size() *2]; int i = 0; + java.util.List names = arg0.getInternalModuleNames(); + + StringBuilder sb = new StringBuilder();arg0.getInternalModule(); + String prev = "."; + for(Name name: names){ + if(!prev.equals(".")) + sb.append("."); + sb.append(name.getInternalId()); + prev = name.getInternalId(); + } + String moduleName = sb.toString(); for(alias n : arg0.getInternalNames()) { + System.err.println("IMPORT FROM " + moduleName +" " + arg0.getInternalModule() + " : " + n.getInternalAsname() + " | " + name(n) + " intName = " + n.getInternalName()); elts[i++] = Ast.makeNode(CAstNode.DECL_STMT, - Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any)), + Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any))); + CAstNode importAst = Ast.makeNode(CAstNode.PRIMITIVE, + Ast.makeConstant("import"), + Ast.makeConstant(moduleName.replace(".", "/"))); + elts[i++] = Ast.makeNode(CAstNode.ASSIGN, + Ast.makeNode(CAstNode.VAR, Ast.makeConstant(name(n))), Ast.makeNode(CAstNode.OBJECT_REF, - importAst(arg0), + importAst, Ast.makeConstant(n.getInternalName()))); } return Ast.makeNode(CAstNode.BLOCK_STMT, elts); } - private CAstNode importAst(ImportFrom arg0) { - java.util.List names = arg0.getInternalModuleNames(); - CAstNode importAst = Ast.makeNode(CAstNode.PRIMITIVE, - Ast.makeConstant("import"), - Ast.makeConstant(names.get(0).getInternalId())); - for(int i = 1; i < names.size(); i++) { - importAst = Ast.makeNode(CAstNode.OBJECT_REF, - importAst, - Ast.makeConstant(names.get(i).getInternalId())); - } - return importAst; - } - @Override public CAstNode visitIndex(Index arg0) throws Exception { return arg0.getInternalValue().accept(this); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ssa/PythonImportFromGetInstruction.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ssa/PythonImportFromGetInstruction.java new file mode 100644 index 000000000..25ff17f17 --- /dev/null +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ssa/PythonImportFromGetInstruction.java @@ -0,0 +1,20 @@ +package com.ibm.wala.cast.python.ssa; + +import com.ibm.wala.ssa.SSAGetInstruction; +import com.ibm.wala.ssa.SymbolTable; +import com.ibm.wala.types.FieldReference; + +public class PythonImportFromGetInstruction extends SSAGetInstruction { + public PythonImportFromGetInstruction(int iindex, int result, int ref, FieldReference field){ + super(iindex, result, ref, field); + } + + public PythonImportFromGetInstruction(int iindex, int result, FieldReference field){ + super(iindex, result, field); + } + + public String toString(SymbolTable symbolTable){ + return getValueString(symbolTable, getDef()) + " = importFromGet " + getDeclaredField() + " " + + getValueString(symbolTable, getRef()); + } +} diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java index 9fda2be88..cde2e1909 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java @@ -52,4 +52,6 @@ public class PythonTypes extends AstTypeReference { public static final TypeReference trampoline = TypeReference.findOrCreate(pythonLoader, TypeName.findOrCreate("Ltrampoline")); + public static final TypeReference module = TypeReference.findOrCreate(pythonLoader, TypeName.findOrCreate("Lmodule")); + } From 0b4af6d4f3f1e857f4ab7dc56884e4039cb58516 Mon Sep 17 00:00:00 2001 From: Sifis Lagouvardos Date: Fri, 8 Feb 2019 15:19:17 +0200 Subject: [PATCH 2/4] Got another case for getting globals to work correctly. --- .../python/ir/PythonCAstToIRTranslator.java | 36 ++++++++++++++++--- .../wala/cast/python/parser/PythonParser.java | 2 +- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java index 30b3098b4..c7dadfbae 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java @@ -32,6 +32,7 @@ import com.ibm.wala.cast.tree.CAstEntity; import com.ibm.wala.cast.tree.CAstNode; import com.ibm.wala.cast.tree.CAstSourcePositionMap.Position; +import com.ibm.wala.cast.tree.CAstSymbol; import com.ibm.wala.cast.tree.CAstType; import com.ibm.wala.cast.tree.impl.CAstOperator; import com.ibm.wala.cast.tree.impl.CAstSymbolImpl; @@ -276,8 +277,8 @@ protected void leaveFunctionEntity(CAstEntity n, WalkContext context, WalkContex CAstVisitor visitor) { super.leaveFunctionEntity(n, context, codeContext, visitor); - String fnName = composeEntityName(context, n) + "_defaults"; - if (n.getArgumentDefaults() != null) { + String fnName = composeEntityName(context, n) + "_defaults"; + if (n.getArgumentDefaults() != null && n.getArgumentDefaults().length > 0) { int first = n.getArgumentCount() - n.getArgumentDefaults().length; for(int i = first; i < n.getArgumentCount(); i++) { CAstNode dflt = n.getArgumentDefaults()[i - first]; @@ -287,6 +288,31 @@ protected void leaveFunctionEntity(CAstEntity n, WalkContext context, WalkContex } } + protected void leaveDeclStmt(CAstNode n, WalkContext c, CAstVisitor visitor) { + CAstSymbol s = (CAstSymbol) n.getChild(0).getValue(); + String nm = s.name(); + CAstType t = s.type(); + Scope scope = c.currentScope(); + if (n.getChildCount() == 2) { + CAstNode v = n.getChild(1); + if(isGlobal(c, nm)){ + scope.declare(s); + doGlobalWrite(c, nm, makeType(t), c.getValue(v)); + } + else if (scope.contains(nm) && scope.lookup(nm).getDefiningScope() == scope) { + assert !s.isFinal(); + doLocalWrite(c, nm, makeType(t), c.getValue(v)); + } else if (v.getKind() != CAstNode.CONSTANT && v.getKind() != CAstNode.VAR && v.getKind() != CAstNode.THIS) { + scope.declare(s, c.getValue(v)); + } else { + scope.declare(s); + doLocalWrite(c, nm, makeType(t), c.getValue(v)); + } + } else { + c.currentScope().declare(s); + } + } + @Override protected void leaveVar(CAstNode n, WalkContext c, CAstVisitor visitor) { WalkContext context = c; @@ -605,7 +631,8 @@ protected boolean doVisit(CAstNode n, WalkContext context, CAstVisitor names = arg0.getInternalModuleNames(); - StringBuilder sb = new StringBuilder();arg0.getInternalModule(); + StringBuilder sb = new StringBuilder(); String prev = "."; for(Name name: names){ if(!prev.equals(".")) From 9673fa948de51e02552d6b004da5629dc5dad245 Mon Sep 17 00:00:00 2001 From: Sifis Lagouvardos Date: Fri, 8 Feb 2019 15:32:43 +0200 Subject: [PATCH 3/4] Remove some prints. --- .../source/com/ibm/wala/cast/python/parser/PythonParser.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java index 2ab62156d..aa457718e 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java @@ -1032,7 +1032,6 @@ public CAstNode visitImport(Import arg0) throws Exception { int i = 0; CAstNode[] elts = new CAstNode[ arg0.getInternalNames().size()*2 ]; for(alias n : arg0.getInternalNames()) { - System.err.println("SIMPLE IMPORT "+ n.getInternalAsname() + " | " + name(n) + " intName = " + n.getInternalName()); elts[i++] = Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any))); elts[i++] = Ast.makeNode(CAstNode.ASSIGN, @@ -1058,7 +1057,6 @@ public CAstNode visitImportFrom(ImportFrom arg0) throws Exception { } String moduleName = sb.toString(); for(alias n : arg0.getInternalNames()) { - System.err.println("IMPORT FROM " + moduleName +" " + arg0.getInternalModule() + " : " + n.getInternalAsname() + " | " + name(n) + " intName = " + n.getInternalName()); elts[i++] = Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any))); CAstNode importAst = Ast.makeNode(CAstNode.PRIMITIVE, From c2cd2e0c3c5fad7475ec1e49a78b3f0f3eaaf2de Mon Sep 17 00:00:00 2001 From: cfish Date: Tue, 14 Apr 2020 16:14:06 +0300 Subject: [PATCH 4/4] Fix bug that made UT-12 crash. --- .../source/com/ibm/wala/cast/python/parser/PythonParser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java index aa457718e..9e05efb7e 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/PythonParser.java @@ -1257,6 +1257,9 @@ private CAstNode doGenerators(java.util.List generators, CAstNode result = Ast.makeNode(CAstNode.BLOCK_EXPR, Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(tempName, PythonCAstToIRTranslator.Any)), c.getInternalIter().accept(this)), + Ast.makeNode(CAstNode.ASSIGN, + c.getInternalTarget().accept(this), + Ast.makeConstant(null)), Ast.makeNode(CAstNode.LOOP, test, Ast.makeNode(CAstNode.BLOCK_EXPR,