我们从Python开源项目中，提取了以下18个代码示例，用于说明如何使用compiler.ast()。
def CheckedEval(file_contents): """Return the eval of a gyp file. The gyp file is restricted to dictionaries and lists only, and repeated keys are not allowed. Note that this is slower than eval() is. """ ast = compiler.parse(file_contents) assert isinstance(ast, Module) c1 = ast.getChildren() assert c1[0] is None assert isinstance(c1[1], Stmt) c2 = c1[1].getChildren() assert isinstance(c2[0], Discard) c3 = c2[0].getChildren() assert len(c3) == 1 return CheckNode(c3[0], [])
def visit_Stmt(self, node): def _check_del(n): # del x is just AssName('x', 'OP_DELETE') # we want to transform it to Delete([Name('x', Del())]) dcls = (_ast.Name, _ast.List, _ast.Subscript, _ast.Attribute) if isinstance(n, dcls) and isinstance(n.ctx, _ast.Del): return self._new(_ast.Delete, [n]) elif isinstance(n, _ast.Tuple) and isinstance(n.ctx, _ast.Del): # unpack last tuple to avoid making del (x, y, z,); # out of del x, y, z; (there's no difference between # this two in compiler.ast) return self._new(_ast.Delete, n.elts) else: return n def _keep(n): if isinstance(n, _ast.Expr) and n.value is None: return False else: return True return [s for s in [_check_del(self.visit(n)) for n in node.nodes] if _keep(s)]
def walk(self, ast): "Validate each node in AST and return True if AST is 'safe'." self.visit(ast) return self.errors == []
def get_args(val): d = {} args = ast.parse("d(" + val + ")", mode='eval').body.args i = 1 for arg in args: if isinstance(arg, ast.Name): d[str(i)] = literal_eval(arg.id) else: d[str(i)] = literal_eval(arg) i += 1 return d
def get_kwargs(val): d = {} args = ast.parse("d(" + val + ")", mode='eval').body.keywords for arg in args: d[arg.arg] = literal_eval(arg.value) return d
def parse_to_list(val): values = ast.parse("[" + val + "]", mode='eval').body.elts return [literal_eval(v) for v in values]
def literal_eval(node_or_string): """ Safely evaluate an expression node or a string containing a Python expression. The string or node provided may only consist of the following Python literal structures: strings, numbers, tuples, lists, dicts, booleans, and None. """ _safe_names = {'None': None, 'True': True, 'False': False} if isinstance(node_or_string, basestring): node_or_string = compiler.parse(node_or_string, mode='eval') if isinstance(node_or_string, Expression): node_or_string = node_or_string.node def _convert(node): if isinstance(node, Const) and isinstance(node.value, (basestring, int, float, long, complex)): return node.value elif isinstance(node, Tuple): return tuple(map(_convert, node.nodes)) elif isinstance(node, compiler.ast.List): return list(map(_convert, node.nodes)) elif isinstance(node, Dict): return dict((_convert(k), _convert(v)) for k, v in node.items) elif isinstance(node, Name): if node.name in _safe_names: return _safe_names[node.name] elif isinstance(node, UnarySub): return -_convert(node.expr) raise ValueError('malformed string') return _convert(node_or_string)
def __init__(self, indent, lineno, body, final): Node.__init__(self, indent, lineno) if isinstance(body, compiler.ast.TryExcept): self.body = transform(indent, lineno, body) self.body.has_finally = True else: self.body = transform(indent + 1, lineno, body) self.final = transform(indent + 1, lineno, final) return
def _extract_args(self, node): tab = node.argnames[:] if node.flags & compiler.ast.CO_VARKEYWORDS: kwarg = tab[-1] tab = tab[:-1] else: kwarg = None if node.flags & compiler.ast.CO_VARARGS: vararg = tab[-1] tab = tab[:-1] else: vararg = None def _tup(t): if isinstance(t, str): return self._new(_ast.Name, t, _ast.Store()) elif isinstance(t, tuple): elts = [_tup(x) for x in t] return self._new(_ast.Tuple, elts, _ast.Store()) else: raise NotImplemented args = [] for arg in tab: if isinstance(arg, str): args.append(self._new(_ast.Name, arg, _ast.Param())) elif isinstance(arg, tuple): args.append(_tup(arg)) else: assert False, node.__class__ defaults = [self.visit(d) for d in node.defaults] return self._new(_ast.arguments, args, vararg, kwarg, defaults)
def visit_CallFunc(self, node): args = [] keywords = [] for arg in node.args: if isinstance(arg, compiler.ast.Keyword): keywords.append(self._new(_ast.keyword, arg.name, self.visit(arg.expr))) else: args.append(self.visit(arg)) return self._new(_ast.Call, self.visit(node.node), args, keywords, self.visit(node.star_args), self.visit(node.dstar_args))
def find_imports(fn, verbose, ignores): "Yields a list of the module names the file 'fn' depends on." ast, _ = parse_python_source(fn) if ast is None: raise StopIteration found_imports = get_ast_imports(ast) if found_imports is None: raise StopIteration dn = dirname(fn) packroot = None for modname, rname, lname, lineno, _, _ in found_imports: islocal = False names = modname.split('.') if find_dotted(names, dn): # This is a local import, we need to find the root in order to # compute the absolute module name. if packroot is None: packroot = find_package_root(fn, ignores) if not packroot: logging.warning( "%d: Could not find package root for local import '%s' from '%s'." % (lineno, modname, fn)) continue reldir = dirname(fn)[len(packroot)+1:] modname = '%s.%s' % (reldir.replace(os.sep, '.'), modname) islocal = True if rname is not None: modname = '%s.%s' % (modname, rname) yield (modname, lineno, islocal)
def parse_python_source(fn): """Parse the file 'fn' and return two things: 1. The AST tree. 2. A list of lines of the source line (typically used for verbose error messages). If the file has a syntax error in it, the first argument will be None. """ # Read the file's contents to return it. # Note: we make sure to use universal newlines. try: contents = open(fn, 'rU').read() lines = contents.splitlines() except (IOError, OSError), e: logging.error("Could not read file '%s'." % fn) return None, None # Convert the file to an AST. try: ast = compiler.parse(contents) except SyntaxError, e: err = '%s:%s: %s' % (fn, e.lineno or '--', e.msg) logging.error("Error processing file '%s':\n%s" % (fn, err)) return None, lines except TypeError, e: # Note: this branch untested, applied from a user-submitted patch. err = '%s: %s' % (fn, str(e)) logging.error("Error processing file '%s':\n%s" % (fn, err)) return None, lines return ast, lines
def get_ast_imports(ast): """ Given an AST, return a list of module tuples for the imports found, in the form: (modname, remote-name, local-name, lineno, pragma) """ assert ast is not None vis = ImportVisitor() compiler.walk(ast, vis, ImportWalker(vis)) found_imports = vis.finalize() return found_imports # **WARNING** This is where all the evil lies. Risk and peril. Watch out.
def printAst(ast, indent=' ', stream=sys.stdout, initlevel=0): "Pretty-print an AST to the given output stream." rec_node(ast, initlevel, indent, stream.write) stream.write('\n')
def safe_eval(code, context = {}, timeout_secs = 5): """ Validate source code and make sure it contains no unauthorized expression/statements as configured via 'unallowed_ast_nodes' and 'unallowed_builtins'. By default this means that code is not allowed import modules or access dangerous builtins like 'open' or 'eval'. If code is considered 'safe' it will be executed via 'exec' using 'context' as the global environment. More details on how code is executed can be found in the Python Reference Manual section 6.14 (ignore the remark on '__builtins__'). The 'context' enviroment is also validated and is not allowed to contain modules or builtins. The following exception will be raised on errors: if 'context' contains unallowed objects = SafeEvalContextException if code is didn't validate and is considered 'unsafe' = SafeEvalCodeException if code did not execute within the given timelimit = SafeEvalTimeoutException """ ctx_errkeys, ctx_errors = [], [] for (key, obj) in context.items(): if inspect.isbuiltin(obj): ctx_errkeys.append(key) ctx_errors.append("key '%s' : unallowed builtin %s" % (key, obj)) if inspect.ismodule(obj): ctx_errkeys.append(key) ctx_errors.append("key '%s' : unallowed module %s" % (key, obj)) if ctx_errors: raise SafeEvalContextException(ctx_errkeys, ctx_errors) ast = compiler.parse(code) checker = SafeEvalVisitor() if checker.walk(ast): exec_timed(code, context, timeout_secs) else: raise SafeEvalCodeException(code, checker.errors) #---------------------------------------------------------------------- # Basic tests. #----------------------------------------------------------------------
def find_dependencies(fn, verbose, process_pragmas, ignore_unused=False, warning_lambda=logging.warning, debug_lambda=logging.debug): "Returns a list of the files 'fn' depends on." file_errors = [] ast, _ = parse_python_source(fn) if ast is None: return [], file_errors found_imports = get_ast_imports(ast) if found_imports is None: return [], file_errors # Filter out the unused imports if requested. if ignore_unused: found_imports, unused_imports = filter_unused_imports(ast, found_imports) for modname, rname, lname, lineno, level, pragma in unused_imports: file_errors.append((ERROR_UNUSED, lname)) output_code = (verbose >= 2) source_lines = None if output_code: source_lines = open(fn, 'rU').read().splitlines() files = [] assert not isdir(fn) dn = dirname(fn) seenset = set() for x in found_imports: mod, rname, lname, lineno, level, pragma = x if process_pragmas and pragma == 'OPTIONAL': if rname is None: msg = WARNING_OPTIONAL % (lineno, mod) else: msg = '%s.%s' % (mod, rname) logging.warning(msg) continue sig = (mod, rname) if sig in seenset: continue seenset.add(sig) modfile, errors = find_dotted_module(mod, rname, dn, level) if errors: file_errors.extend(errors) for err, name in errors: if err is ERROR_IMPORT: efun = warning_lambda else: efun = debug_lambda efun(err % (lineno, name)) if output_code: efun(ERROR_SOURCE % source_lines[lineno-1].rstrip()) if modfile is None: continue files.append(realpath(modfile)) return files, file_errors
