Skip to content

Commit 2c86359

Browse files
youknowoneyouknowone
authored
Merge pull request RustPython#2045 from Lynskylate/feature-symtable
Add is_imported and is_annotation for symtable
2 parents caf419a + 31061b0 commit 2c86359

File tree

3 files changed

+354
-8
lines changed

3 files changed

+354
-8
lines changed

‎Lib/test/test_symtable.py

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
"""
2+
Test the API of the symtable module.
3+
"""
4+
import symtable
5+
import unittest
6+
7+
8+
9+
TEST_CODE = """
10+
import sys
11+
12+
glob = 42
13+
some_var = 12
14+
15+
class Mine:
16+
instance_var = 24
17+
def a_method(p1, p2):
18+
pass
19+
20+
def spam(a, b, *var, **kw):
21+
global bar
22+
bar = 47
23+
some_var = 10
24+
x = 23
25+
glob
26+
def internal():
27+
return x
28+
def other_internal():
29+
nonlocal some_var
30+
some_var = 3
31+
return some_var
32+
return internal
33+
34+
def foo():
35+
pass
36+
37+
def namespace_test(): pass
38+
def namespace_test(): pass
39+
"""
40+
41+
42+
def find_block(block, name):
43+
for ch in block.get_children():
44+
if ch.get_name() == name:
45+
return ch
46+
47+
48+
class SymtableTest(unittest.TestCase):
49+
50+
top = symtable.symtable(TEST_CODE, "?", "exec")
51+
# These correspond to scopes in TEST_CODE
52+
Mine = find_block(top, "Mine")
53+
a_method = find_block(Mine, "a_method")
54+
spam = find_block(top, "spam")
55+
internal = find_block(spam, "internal")
56+
other_internal = find_block(spam, "other_internal")
57+
foo = find_block(top, "foo")
58+
59+
def test_type(self):
60+
self.assertEqual(self.top.get_type(), "module")
61+
self.assertEqual(self.Mine.get_type(), "class")
62+
self.assertEqual(self.a_method.get_type(), "function")
63+
self.assertEqual(self.spam.get_type(), "function")
64+
self.assertEqual(self.internal.get_type(), "function")
65+
66+
@unittest.skip("TODO: RUSTPYTHON")
67+
def test_id(self):
68+
self.assertGreater(self.top.get_id(), 0)
69+
self.assertGreater(self.Mine.get_id(), 0)
70+
self.assertGreater(self.a_method.get_id(), 0)
71+
self.assertGreater(self.spam.get_id(), 0)
72+
self.assertGreater(self.internal.get_id(), 0)
73+
74+
@unittest.skip("TODO: RUSTPYTHON")
75+
def test_optimized(self):
76+
self.assertFalse(self.top.is_optimized())
77+
78+
self.assertTrue(self.spam.is_optimized())
79+
80+
@unittest.skip("TODO: RUSTPYTHON")
81+
def test_nested(self):
82+
self.assertFalse(self.top.is_nested())
83+
self.assertFalse(self.Mine.is_nested())
84+
self.assertFalse(self.spam.is_nested())
85+
self.assertTrue(self.internal.is_nested())
86+
87+
def test_children(self):
88+
self.assertTrue(self.top.has_children())
89+
self.assertTrue(self.Mine.has_children())
90+
self.assertFalse(self.foo.has_children())
91+
92+
def test_lineno(self):
93+
self.assertEqual(self.top.get_lineno(), 0)
94+
self.assertEqual(self.spam.get_lineno(), 12)
95+
96+
@unittest.skip("TODO: RUSTPYTHON")
97+
def test_function_info(self):
98+
func = self.spam
99+
self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"])
100+
expected = ['a', 'b', 'internal', 'kw', 'other_internal', 'some_var', 'var', 'x']
101+
self.assertEqual(sorted(func.get_locals()), expected)
102+
self.assertEqual(sorted(func.get_globals()), ["bar", "glob"])
103+
self.assertEqual(self.internal.get_frees(), ("x",))
104+
105+
@unittest.skip("TODO: RUSTPYTHON")
106+
def test_globals(self):
107+
self.assertTrue(self.spam.lookup("glob").is_global())
108+
self.assertFalse(self.spam.lookup("glob").is_declared_global())
109+
self.assertTrue(self.spam.lookup("bar").is_global())
110+
self.assertTrue(self.spam.lookup("bar").is_declared_global())
111+
self.assertFalse(self.internal.lookup("x").is_global())
112+
self.assertFalse(self.Mine.lookup("instance_var").is_global())
113+
self.assertTrue(self.spam.lookup("bar").is_global())
114+
115+
@unittest.skip("TODO: RUSTPYTHON")
116+
def test_nonlocal(self):
117+
self.assertFalse(self.spam.lookup("some_var").is_nonlocal())
118+
self.assertTrue(self.other_internal.lookup("some_var").is_nonlocal())
119+
expected = ("some_var",)
120+
self.assertEqual(self.other_internal.get_nonlocals(), expected)
121+
122+
def test_local(self):
123+
self.assertTrue(self.spam.lookup("x").is_local())
124+
self.assertFalse(self.spam.lookup("bar").is_local())
125+
126+
def test_free(self):
127+
self.assertTrue(self.internal.lookup("x").is_free())
128+
129+
def test_referenced(self):
130+
self.assertTrue(self.internal.lookup("x").is_referenced())
131+
self.assertTrue(self.spam.lookup("internal").is_referenced())
132+
self.assertFalse(self.spam.lookup("x").is_referenced())
133+
134+
def test_parameters(self):
135+
for sym in ("a", "var", "kw"):
136+
self.assertTrue(self.spam.lookup(sym).is_parameter())
137+
self.assertFalse(self.spam.lookup("x").is_parameter())
138+
139+
def test_symbol_lookup(self):
140+
self.assertEqual(len(self.top.get_identifiers()),
141+
len(self.top.get_symbols()))
142+
143+
self.assertRaises(KeyError, self.top.lookup, "not_here")
144+
145+
def test_namespaces(self):
146+
self.assertTrue(self.top.lookup("Mine").is_namespace())
147+
self.assertTrue(self.Mine.lookup("a_method").is_namespace())
148+
self.assertTrue(self.top.lookup("spam").is_namespace())
149+
self.assertTrue(self.spam.lookup("internal").is_namespace())
150+
self.assertTrue(self.top.lookup("namespace_test").is_namespace())
151+
self.assertFalse(self.spam.lookup("x").is_namespace())
152+
153+
# TODO(RUSTPYTHON): lookup should return same pythonref
154+
# self.assertTrue(self.top.lookup("spam").get_namespace() is self.spam)
155+
ns_test = self.top.lookup("namespace_test")
156+
self.assertEqual(len(ns_test.get_namespaces()), 2)
157+
self.assertRaises(ValueError, ns_test.get_namespace)
158+
159+
def test_assigned(self):
160+
self.assertTrue(self.spam.lookup("x").is_assigned())
161+
self.assertTrue(self.spam.lookup("bar").is_assigned())
162+
self.assertTrue(self.top.lookup("spam").is_assigned())
163+
self.assertTrue(self.Mine.lookup("a_method").is_assigned())
164+
self.assertFalse(self.internal.lookup("x").is_assigned())
165+
166+
def test_annotated(self):
167+
st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
168+
st2 = st1.get_children()[0]
169+
self.assertTrue(st2.lookup('x').is_local())
170+
self.assertTrue(st2.lookup('x').is_annotated())
171+
self.assertFalse(st2.lookup('x').is_global())
172+
st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec')
173+
st4 = st3.get_children()[0]
174+
self.assertTrue(st4.lookup('x').is_local())
175+
self.assertFalse(st4.lookup('x').is_annotated())
176+
177+
# Test that annotations in the global scope are valid after the
178+
# variable is declared as nonlocal.
179+
st5 = symtable.symtable('global x\nx: int', 'test', 'exec')
180+
self.assertTrue(st5.lookup("x").is_global())
181+
182+
# Test that annotations for nonlocals are valid after the
183+
# variable is declared as nonlocal.
184+
st6 = symtable.symtable('def g():\n'
185+
' x = 2\n'
186+
' def f():\n'
187+
' nonlocal x\n'
188+
' x: int',
189+
'test', 'exec')
190+
191+
def test_imported(self):
192+
self.assertTrue(self.top.lookup("sys").is_imported())
193+
194+
def test_name(self):
195+
self.assertEqual(self.top.get_name(), "top")
196+
self.assertEqual(self.spam.get_name(), "spam")
197+
self.assertEqual(self.spam.lookup("x").get_name(), "x")
198+
self.assertEqual(self.Mine.get_name(), "Mine")
199+
200+
@unittest.skip("TODO: RUSTPYTHON")
201+
def test_class_info(self):
202+
self.assertEqual(self.Mine.get_methods(), ('a_method',))
203+
204+
@unittest.skip("TODO: RUSTPYTHON")
205+
def test_filename_correct(self):
206+
### Bug tickler: SyntaxError file name correct whether error raised
207+
### while parsing or building symbol table.
208+
def checkfilename(brokencode, offset):
209+
try:
210+
symtable.symtable(brokencode, "spam", "exec")
211+
except SyntaxError as e:
212+
self.assertEqual(e.filename, "spam")
213+
self.assertEqual(e.lineno, 1)
214+
self.assertEqual(e.offset, offset)
215+
else:
216+
self.fail("no SyntaxError for %r" % (brokencode,))
217+
# TODO: RUSTPYTHON, now offset get 15
218+
checkfilename("def f(x): foo)(", 14) # parse-time
219+
checkfilename("def f(x): global x", 11) # symtable-build-time
220+
symtable.symtable("pass", b"spam", "exec")
221+
with self.assertWarns(DeprecationWarning), \
222+
self.assertRaises(TypeError):
223+
symtable.symtable("pass", bytearray(b"spam"), "exec")
224+
with self.assertWarns(DeprecationWarning):
225+
symtable.symtable("pass", memoryview(b"spam"), "exec")
226+
with self.assertRaises(TypeError):
227+
symtable.symtable("pass", list(b"spam"), "exec")
228+
229+
def test_eval(self):
230+
symbols = symtable.symtable("42", "?", "eval")
231+
232+
@unittest.skip("TODO: RUSTPYTHON")
233+
def test_single(self):
234+
symbols = symtable.symtable("42", "?", "single")
235+
236+
def test_exec(self):
237+
symbols = symtable.symtable("def f(x): return x", "?", "exec")
238+
239+
@unittest.skip("TODO: RUSTPYTHON")
240+
def test_bytes(self):
241+
top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec")
242+
self.assertIsNotNone(find_block(top, "Mine"))
243+
244+
code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n'
245+
246+
top = symtable.symtable(code, "?", "exec")
247+
self.assertIsNotNone(find_block(top, "\u017d"))
248+
249+
250+
if __name__ == '__main__':
251+
unittest.main()

‎compiler/src/symboltable.rs

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,13 @@ pub struct Symbol {
9797
pub name: String,
9898
// pub table: SymbolTableRef,
9999
pub scope: SymbolScope,
100+
// TODO: Use bitflags replace
100101
pub is_referenced: bool,
101102
pub is_assigned: bool,
102103
pub is_parameter: bool,
103104
pub is_free: bool,
105+
pub is_annotated: bool,
106+
pub is_imported: bool,
104107

105108
// indicates if the symbol gets a value assigned by a named expression in a comprehension
106109
// this is required to correct the scope in the analysis.
@@ -121,6 +124,8 @@ impl Symbol {
121124
is_assigned: false,
122125
is_parameter: false,
123126
is_free: false,
127+
is_annotated: false,
128+
is_imported: false,
124129
is_assign_namedexpr_in_comprehension: false,
125130
is_iter: false,
126131
}
@@ -394,7 +399,10 @@ enum SymbolUsage {
394399
Nonlocal,
395400
Used,
396401
Assigned,
402+
Imported,
403+
AnnotationAssigned,
397404
Parameter,
405+
AnnotationParameter,
398406
AssignedNamedExprInCompr,
399407
Iter,
400408
}
@@ -460,7 +468,12 @@ impl SymbolTableBuilder {
460468
}
461469

462470
fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
463-
self.register_name(&parameter.arg, SymbolUsage::Parameter)
471+
let usage = if parameter.annotation.is_some() {
472+
SymbolUsage::AnnotationParameter
473+
} else {
474+
SymbolUsage::Parameter
475+
};
476+
self.register_name(&parameter.arg, usage)
464477
}
465478

466479
fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@@ -564,12 +577,12 @@ impl SymbolTableBuilder {
564577
for name in names {
565578
if let Some(alias) = &name.alias {
566579
// `import mymodule as myalias`
567-
self.register_name(alias, SymbolUsage::Assigned)?;
580+
self.register_name(alias, SymbolUsage::Imported)?;
568581
} else {
569582
// `import module`
570583
self.register_name(
571584
name.symbol.split('.').next().unwrap(),
572-
SymbolUsage::Assigned,
585+
SymbolUsage::Imported,
573586
)?;
574587
}
575588
}
@@ -601,7 +614,12 @@ impl SymbolTableBuilder {
601614
annotation,
602615
value,
603616
} => {
604-
self.scan_expression(target, &ExpressionContext::Store)?;
617+
// https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
618+
if let ast::ExpressionType::Identifier { ref name } = target.node {
619+
self.register_name(name, SymbolUsage::AnnotationAssigned)?;
620+
} else {
621+
self.scan_expression(target, &ExpressionContext::Store)?;
622+
}
605623
self.scan_expression(annotation, &ExpressionContext::Load)?;
606624
if let Some(value) = value {
607625
self.scan_expression(value, &ExpressionContext::Load)?;
@@ -984,9 +1002,21 @@ impl SymbolTableBuilder {
9841002
});
9851003
}
9861004
}
1005+
SymbolUsage::Imported => {
1006+
symbol.is_assigned = true;
1007+
symbol.is_imported = true;
1008+
}
9871009
SymbolUsage::Parameter => {
9881010
symbol.is_parameter = true;
9891011
}
1012+
SymbolUsage::AnnotationParameter => {
1013+
symbol.is_parameter = true;
1014+
symbol.is_annotated = true;
1015+
}
1016+
SymbolUsage::AnnotationAssigned => {
1017+
symbol.is_assigned = true;
1018+
symbol.is_annotated = true;
1019+
}
9901020
SymbolUsage::Assigned => {
9911021
symbol.is_assigned = true;
9921022
}

0 commit comments

Comments
 (0)