Skip to content

Commit 6c6e35f

Browse files
markshannonmarkshannon
committed
Python: Enhance points-to to support type-hint analysis.
1 parent 3fab514 commit 6c6e35f

18 files changed

+193
-4
lines changed

‎python/ql/src/semmle/python/objects/Classes.qll

+78
Original file line numberDiff line numberDiff line change
@@ -365,3 +365,81 @@ class DynamicallyCreatedClass extends ClassObjectInternal, TDynamicClass {
365365

366366
}
367367

368+
class SubscriptedTypeInternal extends ObjectInternal, TSubscriptedType {
369+
370+
ObjectInternal getGeneric() {
371+
this = TSubscriptedType(result, _)
372+
}
373+
374+
ObjectInternal getSpecializer() {
375+
this = TSubscriptedType(_, result)
376+
}
377+
378+
override string getName() { result = this.getGeneric().getName() }
379+
380+
override string toString() { result = this.getGeneric().toString() + "[" + this.getSpecializer().toString() + "]" }
381+
382+
override predicate introducedAt(ControlFlowNode node, PointsToContext context) {
383+
exists(ObjectInternal generic, ObjectInternal index |
384+
this = TSubscriptedType(generic, index) and
385+
Expressions::subscriptPartsPointsTo(node, context, generic, index)
386+
)
387+
}
388+
389+
/** Gets the class declaration for this object, if it is a class with a declaration. */
390+
override ClassDecl getClassDeclaration() {
391+
result = this.getGeneric().getClassDeclaration()
392+
}
393+
394+
/** True if this "object" is a class. That is, its class inherits from `type` */
395+
override boolean isClass() { result = true }
396+
397+
override ObjectInternal getClass() {
398+
result = this.getGeneric().getClass()
399+
}
400+
401+
override predicate notTestableForEquality() { none() }
402+
403+
override Builtin getBuiltin() { none() }
404+
405+
override ControlFlowNode getOrigin() { none() }
406+
407+
override predicate callResult(ObjectInternal obj, CfgOrigin origin) { none() }
408+
409+
override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { none() }
410+
411+
override predicate calleeAndOffset(Function scope, int paramOffset){ none() }
412+
413+
override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { none() }
414+
415+
override predicate attributesUnknown() { none() }
416+
417+
override boolean isDescriptor() { result = false }
418+
419+
override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() }
420+
421+
override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() }
422+
423+
override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() }
424+
425+
override int length() { none() }
426+
427+
override boolean booleanValue() { result = true }
428+
429+
override int intValue() { none()}
430+
431+
override string strValue() { none() }
432+
433+
override predicate subscriptUnknown() { none() }
434+
435+
override predicate contextSensitiveCallee() { none() }
436+
437+
override predicate useOriginAsLegacyObject() { none() }
438+
439+
/* Classes aren't usually iterable, but can e.g. Enums */
440+
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
441+
442+
}
443+
444+
445+

‎python/ql/src/semmle/python/objects/Instances.qll

+8-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,14 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
9191
override predicate notTestableForEquality() { none() }
9292

9393
override ObjectInternal getClass() {
94-
this = TSpecificInstance(_, result, _)
94+
exists(ClassObjectInternal cls, ClassDecl decl |
95+
this = TSpecificInstance(_, cls, _) and
96+
decl = cls.getClassDeclaration() |
97+
if decl.callReturnsInstance() then
98+
result = cls
99+
else
100+
result = TUnknownClass()
101+
)
95102
}
96103

97104
/** Gets the `Builtin` for this object, if any.

‎python/ql/src/semmle/python/objects/ObjectAPI.qll

+5-1
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,11 @@ class ClassValue extends Value {
322322

323323
/** Gets an improper super type of this class. */
324324
ClassValue getASuperType() {
325-
result = Types::getMro(this).getAnItem()
325+
result = this.getBaseType(_)
326+
or
327+
result = this.getASuperType().getBaseType(_)
328+
or
329+
result = this
326330
}
327331

328332
/** Looks up the attribute `name` on this class.

‎python/ql/src/semmle/python/objects/ObjectInternal.qll

+2
Original file line numberDiff line numberDiff line change
@@ -557,6 +557,8 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
557557

558558
override string toString() {
559559
result = "Decorated " + this.decoratedObject().toString()
560+
or
561+
not exists(this.decoratedObject()) and result = "Decorated function"
560562
}
561563

562564
override boolean booleanValue() { result = true }

‎python/ql/src/semmle/python/objects/Sequences.qll

+8-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,14 @@ abstract class TupleObjectInternal extends SequenceObjectInternal {
4949
or
5050
n = 3 and this.length() > 3 and result = (this.length()-3).toString() + " more..."
5151
or
52-
result = this.getItem(n).toString() + ", " + this.contents(n+1)
52+
result = this.item(n) + ", " + this.contents(n+1)
53+
}
54+
55+
private string item(int n) {
56+
result = this.getItem(n).toString()
57+
or
58+
n in [0..this.length()-1] and
59+
not exists(this.getItem(n)) and result = "?"
5360
}
5461

5562
/** Gets the class declaration for this object, if it is a declared class. */

‎python/ql/src/semmle/python/objects/TObject.qll

+13-1
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ cached newtype TObject =
132132
/* An instance of `cls`, instantiated at `instantiation` given the `context`. */
133133
TSpecificInstance(ControlFlowNode instantiation, ClassObjectInternal cls, PointsToContext context) {
134134
PointsToInternal::pointsTo(instantiation.(CallNode).getFunction(), context, cls, _) and
135-
cls.isSpecial() = false and cls.getClassDeclaration().callReturnsInstance()
135+
cls.isSpecial() = false
136136
or
137137
literal_instantiation(instantiation, cls, context)
138138
}
@@ -236,6 +236,18 @@ cached newtype TObject =
236236
TDecoratedFunction(CallNode call) {
237237
call.isFunctionDecoratorCall()
238238
}
239+
or
240+
/* Represents a subscript operation applied to a type. For type-hint analysis */
241+
TSubscriptedType(ObjectInternal generic, ObjectInternal index) {
242+
isType(generic) and
243+
Expressions::subscriptPartsPointsTo(_, _, generic, index)
244+
}
245+
246+
predicate isType(ObjectInternal t) {
247+
t.isClass() = true
248+
or
249+
t.getOrigin().getEnclosingModule().getName().matches("%typing")
250+
}
239251

240252
private predicate is_power_2(int n) {
241253
n = 1 or

‎python/ql/src/semmle/python/pointsto/PointsTo.qll

+7
Original file line numberDiff line numberDiff line change
@@ -1229,6 +1229,13 @@ module Expressions {
12291229
origin = subscr
12301230
}
12311231

1232+
predicate subscriptPartsPointsTo(SubscriptNode subscr, PointsToContext context, ObjectInternal objvalue, ObjectInternal indexvalue) {
1233+
exists(ControlFlowNode index |
1234+
subscriptObjectAndIndex(subscr, context, _, objvalue, index) and
1235+
PointsToInternal::pointsTo(index, context, indexvalue, _)
1236+
)
1237+
}
1238+
12321239
pragma [noinline]
12331240
private predicate subscriptObjectAndIndex(SubscriptNode subscr, PointsToContext context, ControlFlowNode obj, ObjectInternal objvalue, ControlFlowNode index) {
12341241
subscr.isLoad() and

‎python/ql/test/3/library-tests/PointsTo/typehints/Values.expected

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
| test.py:1:6:1:11 | test.py:1 | ControlFlowNode for ImportExpr | import | ../../lib/typing.py:0:0:0:0 | Module typing |
2+
| test.py:1:6:1:11 | test.py:1 | ControlFlowNode for ImportExpr | import | ../../lib/typing.py:0:0:0:0 | Module typing |
3+
| test.py:1:20:1:27 | test.py:1 | ControlFlowNode for ImportMember | import | ../../lib/typing.py:18:12:18:32 | _Optional() |
4+
| test.py:1:30:1:32 | test.py:1 | ControlFlowNode for ImportMember | import | ../../lib/typing.py:23:1:23:23 | class Set |
5+
| test.py:3:1:3:32 | test.py:3 | ControlFlowNode for FunctionExpr | import | test.py:3:1:3:32 | Function foo |
6+
| test.py:3:11:3:18 | test.py:3 | ControlFlowNode for Optional | import | ../../lib/typing.py:18:12:18:32 | _Optional() |
7+
| test.py:3:11:3:23 | test.py:3 | ControlFlowNode for Subscript | import | file://:0:0:0:0 | _Optional()[builtin-class int] |
8+
| test.py:3:20:3:22 | test.py:3 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
9+
| test.py:3:29:3:31 | test.py:3 | ControlFlowNode for int | import | file://:0:0:0:0 | builtin-class int |
10+
| test.py:6:1:6:20 | test.py:6 | ControlFlowNode for FunctionExpr | import | test.py:6:1:6:20 | Function bar |
11+
| test.py:6:11:6:13 | test.py:6 | ControlFlowNode for set | import | file://:0:0:0:0 | builtin-class set |
12+
| test.py:6:17:6:19 | test.py:6 | ControlFlowNode for Set | import | ../../lib/typing.py:23:1:23:23 | class Set |

‎python/ql/test/3/library-tests/PointsTo/typehints/Values.ql

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
import python
3+
4+
from ControlFlowNode f, Context ctx, Value v, ControlFlowNode origin
5+
where
6+
f.pointsTo(ctx, v, origin) and
7+
f.getLocation().getFile().getBaseName() = "test.py"
8+
select f.getLocation(), f.toString(), ctx, v

‎python/ql/test/3/library-tests/PointsTo/typehints/options

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
semmle-extractor-options: -p ../../lib/ --max-import-depth=3
2+
optimize: true

‎python/ql/test/3/library-tests/PointsTo/typehints/test.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from typing import Optional, Set
2+
3+
def foo(x:Optional[int]) -> int:
4+
pass
5+
6+
def bar(s:set)->Set:
7+
pass

‎python/ql/test/3/library-tests/lib/typing.py

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#Fake typing module for testing.
2+
3+
class ComplexMetaclass(type):
4+
5+
def __new__(self):
6+
pass
7+
8+
class ComplexBaseClass(metaclass=ComplexMetaclass):
9+
10+
def __new__(self):
11+
pass
12+
13+
class _Optional(ComplexBaseClass, extras=...):
14+
15+
def __new__(self):
16+
pass
17+
18+
Optional = _Optional("Optional")
19+
20+
class Collections(ComplexBaseClass, extras=...):
21+
pass
22+
23+
class Set(Collections):
24+
pass
25+
26+
class List(Collections):
27+
pass
28+
29+
Optional

‎python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected

+2
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@
143143
| Module pointsto_test | 161 | ControlFlowNode for ClassExpr | class Derived3 |
144144
| Module pointsto_test | 161 | ControlFlowNode for Derived3 | class Derived3 |
145145
| Module pointsto_test | 164 | ControlFlowNode for Base | class Base |
146+
| Module pointsto_test | 164 | ControlFlowNode for Base() | Base() |
147+
| Module pointsto_test | 164 | ControlFlowNode for thing | Base() |
146148
| Module pointsto_test | 167 | ControlFlowNode for FunctionExpr | Function multiple_assignment |
147149
| Module pointsto_test | 167 | ControlFlowNode for multiple_assignment | Function multiple_assignment |
148150
| Module pointsto_test | 173 | ControlFlowNode for Base2 | class Base2 |

‎python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected

+2
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,8 @@
233233
| 161 | ControlFlowNode for ClassExpr | class Derived3 |
234234
| 161 | ControlFlowNode for Derived3 | class Derived3 |
235235
| 164 | ControlFlowNode for Base | class Base |
236+
| 164 | ControlFlowNode for Base() | Base() |
237+
| 164 | ControlFlowNode for thing | Base() |
236238
| 167 | ControlFlowNode for FunctionExpr | Function multiple_assignment |
237239
| 167 | ControlFlowNode for multiple_assignment | Function multiple_assignment |
238240
| 168 | ControlFlowNode for Tuple | Tuple |

‎python/ql/test/library-tests/PointsTo/new/NameSpace.expected

+1
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
| h_classes.py:0 | Module code.h_classes | f | Function f |
8080
| h_classes.py:0 | Module code.h_classes | k | Function k |
8181
| h_classes.py:0 | Module code.h_classes | sys | Module sys |
82+
| h_classes.py:0 | Module code.h_classes | thing | Base() |
8283
| h_classes.py:3 | Class C | __init__ | Function __init__ |
8384
| h_classes.py:3 | Class C | x | 'C_x' |
8485
| h_classes.py:23 | Class Base | __init__ | Function __init__ |

‎python/ql/test/library-tests/PointsTo/new/PointsToWithContext.expected

+2
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,8 @@ WARNING: Predicate points_to has been deprecated and may be removed in future (P
403403
| h_classes.py:39 | ControlFlowNode for ClassExpr | class Derived3 | builtin-class type | 39 | import |
404404
| h_classes.py:39 | ControlFlowNode for Derived3 | class Derived3 | builtin-class type | 39 | import |
405405
| h_classes.py:42 | ControlFlowNode for Base | class Base | builtin-class type | 23 | import |
406+
| h_classes.py:42 | ControlFlowNode for Base() | Base() | *UNKNOWN TYPE* | 42 | import |
407+
| h_classes.py:42 | ControlFlowNode for thing | Base() | *UNKNOWN TYPE* | 42 | import |
406408
| h_classes.py:45 | ControlFlowNode for FunctionExpr | Function f | builtin-class function | 45 | import |
407409
| h_classes.py:45 | ControlFlowNode for f | Function f | builtin-class function | 45 | import |
408410
| h_classes.py:48 | ControlFlowNode for ClassExpr | class D | builtin-class type | 48 | import |

‎python/ql/test/library-tests/PointsTo/new/PointsToWithType.expected

+2
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,8 @@ WARNING: Predicate points_to has been deprecated and may be removed in future (P
500500
| h_classes.py:39 | ControlFlowNode for ClassExpr | class Derived3 | builtin-class type | 39 |
501501
| h_classes.py:39 | ControlFlowNode for Derived3 | class Derived3 | builtin-class type | 39 |
502502
| h_classes.py:42 | ControlFlowNode for Base | class Base | builtin-class type | 23 |
503+
| h_classes.py:42 | ControlFlowNode for Base() | Base() | *UNKNOWN TYPE* | 42 |
504+
| h_classes.py:42 | ControlFlowNode for thing | Base() | *UNKNOWN TYPE* | 42 |
503505
| h_classes.py:45 | ControlFlowNode for FunctionExpr | Function f | builtin-class function | 45 |
504506
| h_classes.py:45 | ControlFlowNode for f | Function f | builtin-class function | 45 |
505507
| h_classes.py:48 | ControlFlowNode for ClassExpr | class D | builtin-class type | 48 |

‎python/ql/test/library-tests/PointsTo/new/Values.expected

+5
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111
| a_simple.py:16 | ControlFlowNode for d | runtime | instance of dict | builtin-class dict |
1212
| a_simple.py:18 | ControlFlowNode for FunctionExpr | import | Function multi_loop | builtin-class function |
1313
| a_simple.py:19 | ControlFlowNode for None | runtime | None | builtin-class NoneType |
14+
| a_simple.py:20 | ControlFlowNode for Tuple | runtime | (?, ?, ) | builtin-class tuple |
1415
| a_simple.py:23 | ControlFlowNode for FunctionExpr | import | Function with_definition | builtin-class function |
1516
| a_simple.py:27 | ControlFlowNode for FunctionExpr | import | Function multi_loop_in_try | builtin-class function |
17+
| a_simple.py:29 | ControlFlowNode for Tuple | runtime | (?, ?, ) | builtin-class tuple |
1618
| a_simple.py:31 | ControlFlowNode for KeyError | runtime | builtin-class KeyError | builtin-class type |
1719
| a_simple.py:34 | ControlFlowNode for FunctionExpr | import | Function f | builtin-class function |
1820
| a_simple.py:35 | ControlFlowNode for IntegerLiteral | runtime | int 0 | builtin-class int |
@@ -36,11 +38,14 @@
3638
| a_simple.py:40 | ControlFlowNode for Tuple | runtime | (Unknown value, Unknown value, Unknown value, ) | builtin-class tuple |
3739
| a_simple.py:40 | ControlFlowNode for b | runtime | 'b' | builtin-class str |
3840
| a_simple.py:40 | ControlFlowNode for c | runtime | 'c' | builtin-class str |
41+
| a_simple.py:41 | ControlFlowNode for Tuple | runtime | (?, ?, ?, ) | builtin-class tuple |
3942
| a_simple.py:41 | ControlFlowNode for t | runtime | (int 1, int 2, int 3, ) | builtin-class tuple |
43+
| a_simple.py:42 | ControlFlowNode for Tuple | runtime | (?, ?, ?, ) | builtin-class tuple |
4044
| a_simple.py:42 | ControlFlowNode for w | runtime | (Unknown value, 'b', 'c', ) | builtin-class tuple |
4145
| a_simple.py:42 | ControlFlowNode for w | runtime | (Unknown value, 'b', Unknown value, ) | builtin-class tuple |
4246
| a_simple.py:42 | ControlFlowNode for w | runtime | (Unknown value, Unknown value, 'c', ) | builtin-class tuple |
4347
| a_simple.py:42 | ControlFlowNode for w | runtime | (Unknown value, Unknown value, Unknown value, ) | builtin-class tuple |
48+
| a_simple.py:49 | ControlFlowNode for Tuple | runtime | (?, ?, ?, ) | builtin-class tuple |
4449
| a_simple.py:49 | ControlFlowNode for Tuple | runtime | (Unknown value, 'b', 'c', ) | builtin-class tuple |
4550
| a_simple.py:49 | ControlFlowNode for Tuple | runtime | (Unknown value, 'b', Unknown value, ) | builtin-class tuple |
4651
| a_simple.py:49 | ControlFlowNode for Tuple | runtime | (Unknown value, Unknown value, 'c', ) | builtin-class tuple |

0 commit comments

Comments
 (0)