diff --git a/CppHeaderParser/CppHeaderParser.py b/CppHeaderParser/CppHeaderParser.py index 2825efb..c1e7af3 100644 --- a/CppHeaderParser/CppHeaderParser.py +++ b/CppHeaderParser/CppHeaderParser.py @@ -142,10 +142,6 @@ def trace_print(*args): ignoreSymbols = ["Q_OBJECT"] -# Track what was added in what order and at what depth -parseHistory = [] - - def is_namespace(nameStack): """Determines if a namespace is being specified""" if len(nameStack) == 0: @@ -419,21 +415,59 @@ def _fix_classname(self): return s -def _consume_parens(stack): - i = 0 +_end_balanced_items = {">", "}", "]", ")", "]]"} +_start_balanced_items = { + "<": ">", + "{": "}", + "(": ")", + "[": "]", + "[[": "]]", +} + + +def _consume_balanced_items(stack, init_expected, i): + """ + identical to consume_balanced_tokens, but works on a stack instead + TODO: make them the same function + + :param stack: Stack of tokens to search + :param init_expected: expected token to balance + :param i: Position in stack of initial token + + :returns: position of next token after balanced token + """ + match_stack = deque((init_expected,)) sl = len(stack) - nested = 1 - while i < sl: - t = stack[i] + + while True: i += 1 - if t == ")": - nested -= 1 - if nested == 0: - return i - elif t == "(": - nested += 1 + if i >= sl: + errmsg = "Did not find matching '%s'" % init_expected + raise CppParseError(errmsg) + + tok = stack[i] + if tok in _end_balanced_items: + expected = match_stack.pop() + if tok != expected: + # hack: ambiguous right-shift issues here, really + # should be looking at the context + if tok == ">": + i += 1 + if i < sl and stack[i] == ">": + match_stack.append(expected) + continue - raise CppParseError("Unmatched (") + errmsg = "Expected '%s', found '%s'" % (expected, tok) + raise CppParseError(errmsg) + + if len(match_stack) == 0: + return i + 1 + + continue + + next_end = _start_balanced_items.get(tok) + if next_end: + match_stack.append(next_end) def _parse_template_decl(stack): @@ -472,7 +506,7 @@ def _parse_template_decl(stack): require_ending = True elif t == "(": s = stack[i:] - n = _consume_parens(s) + n = _consume_balanced_items(s, ")", -1) i += n param["param"] = param["param"] + "".join(s[:n]) else: @@ -583,7 +617,7 @@ def _parse_cpp_base(stack, default_access): if t == "(": s = stack[i:] - n = _consume_parens(s) + n = _consume_balanced_items(s, ")", -1) i += n base["decl_name"] = base["decl_name"] + "".join(s[:n]) elif t == "...": @@ -1472,7 +1506,6 @@ def initextra(self): self.stack = ( [] ) # full name stack, good idea to keep both stacks? (simple stack and full stack) - self._classes_brace_level = {} # class name : level self._forward_decls = [] self._template_typenames = [] # template @@ -2195,6 +2228,9 @@ def parse_method_type(self, stack): info["pure_virtual"] = True elif stack[-2] == "delete": info["deleted"] = True + elif stack[-2] == "default": + info["default"] = True + info["defined"] = True r = header.split() name = None @@ -2236,15 +2272,9 @@ def parse_method_type(self, stack): if name.startswith("~"): info["destructor"] = True - if "default;" in stack: - info["defined"] = True - info["default"] = True name = name[1:] - elif not a or (name == self.curClass and len(self.curClass)): + elif not a or (self.curClass and name == self.curClass["name"]): info["constructor"] = True - if "default;" in stack: - info["defined"] = True - info["default"] = True info["name"] = name @@ -2318,15 +2348,15 @@ def _evaluate_method_stack(self): newMethod["path"] = klass["name"] elif self.curClass: # normal case + klass = self.curClass newMethod = CppMethod( self.nameStack, - self.curClass, + klass["name"], info, self.curTemplate, self._get_stmt_doxygen(), self._get_location(self.nameStack), ) - klass = self.classes[self.curClass] klass["methods"][self.curAccessSpecifier].append(newMethod) newMethod["parent"] = klass if klass["namespace"]: @@ -2345,14 +2375,6 @@ def _evaluate_method_stack(self): ) newMethod["parent"] = None self.functions.append(newMethod) - global parseHistory - parseHistory.append( - { - "braceDepth": self.braceDepth, - "item_type": "method", - "item": newMethod, - } - ) else: trace_print("free function?", self.nameStack) @@ -2409,38 +2431,86 @@ def _evaluate_typedef(self): if name not in self.typedefs_order: self.typedefs_order.append(name) + def _finish_struct_typedef(self, toks): + # Look for the name of a typedef struct: struct typedef {...] StructName; or unions to get renamed + debug_print("finish struct typedef") + self.typedef_encountered = False + + # grab the first name token + for tok in toks: + if tok.type == "NAME": + new_name = tok.value + break + else: + return + + # TODO: typedef struct{} X, *PX; + type_name_to_rename = self.curClass["name"] + type_to_rename = self.classes[type_name_to_rename] + type_to_rename["name"] = new_name + # Now re install it in its new location + self.classes[new_name] = type_to_rename + if new_name != type_name_to_rename: + del self.classes[type_name_to_rename] + + def _finish_class_def(self): + + # starting at the last } of a class/struct/union + debug_print("finish_class_def") + + # consume any names for parsing + toks = self._consume_up_to([], ";") + + is_typedef = self.typedef_encountered + if is_typedef: + self._finish_struct_typedef(toks) + + thisClass = self.curClass + self.curClass = thisClass["parent"] + + if not is_typedef: + if len(toks) > 1: + # Deal with "struct { } x;" style of things + expected_types = {",", "NAME", "*", ";"} + stack = [thisClass["name"]] + for tok in toks: + stack.append(tok.value) + if tok.type not in expected_types: + self._parse_error((tok,), ",".join(expected_types)) + + self.nameStack = stack[:-1] + self.stack = stack + self.stmtTokens = toks + + self._evaluate_property_stack(clearStack=False) + + elif self.curClass and thisClass["name"].startswith("<"): + # anonymous class struct/union + stack = [thisClass["name"], "", ";"] + self.nameStack = stack[:-1] + self.stack = stack + + self._evaluate_property_stack(clearStack=False) + + self.stack = [] + self.nameStack = [] + self.stmtTokens = [] + def _evaluate_property_stack(self, clearStack=True, addToVar=None): """Create a Property out of the name stack""" - global parseHistory - debug_print("trace") + debug_print("evaluate_property_stack") if self.nameStack[0] == "typedef": assert self.stack and self.stack[-1] == ";" if self.curClass: typedef = self._parse_typedef(self.stack) name = typedef["name"] - klass = self.classes[self.curClass] - klass["typedefs"][self.curAccessSpecifier].append(name) + self.curClass["typedefs"][self.curAccessSpecifier].append(name) if self.curAccessSpecifier == "public": - klass._public_typedefs[name] = typedef["type"] - Resolver.SubTypedefs[name] = self.curClass + self.curClass._public_typedefs[name] = typedef["type"] + Resolver.SubTypedefs[name] = self.curClass["name"] else: assert 0 elif self.curClass: - if len(self.nameStack) == 1: - # See if we can de anonymize the type - filteredParseHistory = [ - h for h in parseHistory if h["braceDepth"] == self.braceDepth - ] - if ( - len(filteredParseHistory) - and filteredParseHistory[-1]["item_type"] == "class" - ): - self.nameStack.insert(0, filteredParseHistory[-1]["item"]["name"]) - debug_print( - "DEANONYMOIZING %s to type '%s'", - self.nameStack[1], - self.nameStack[0], - ) if "," in self.nameStack: # Maybe we have a variable list # Figure out what part is the variable separator but remember templates of function pointer # First find left most comma outside of a > and ) @@ -2475,13 +2545,10 @@ def _evaluate_property_stack(self, clearStack=True, addToVar=None): ) newVar["namespace"] = self.current_namespace() if self.curClass: - klass = self.classes[self.curClass] + klass = self.curClass klass["properties"][self.curAccessSpecifier].append(newVar) newVar["property_of_class"] = klass["name"] newVar["parent"] = klass - parseHistory.append( - {"braceDepth": self.braceDepth, "item_type": "variable", "item": newVar} - ) if addToVar: newVar.update(addToVar) else: @@ -2554,27 +2621,26 @@ def _evaluate_class_stack(self): classKey = newClass["name"] if parent: - newClass["namespace"] = self.classes[parent]["namespace"] + "::" + parent - newClass["parent"] = self.classes[parent] + newClass["namespace"] = parent["namespace"] + "::" + parent["name"] + newClass["parent"] = parent newClass["access_in_parent"] = self.accessSpecifierStack[-1] - self.classes[parent]["nested_classes"].append(newClass) + parent["nested_classes"].append(newClass) ## supports nested classes with the same name ## - self.curClass = key = parent + "::" + classKey - self._classes_brace_level[key] = self.braceDepth + key = parent["name"] + "::" + classKey elif newClass["parent"]: # nested class defined outside of parent. A::B {...} pcls = newClass["parent"] - parent = pcls["name"] - newClass["namespace"] = pcls["namespace"] + "::" + parent + parentName = pcls["name"] + newClass["namespace"] = pcls["namespace"] + "::" + parentName pcls["nested_classes"].append(newClass) ## supports nested classes with the same name ## - self.curClass = key = parent + "::" + classKey - self._classes_brace_level[key] = self.braceDepth + key = parentName + "::" + classKey else: newClass["namespace"] = self.cur_namespace() - self.curClass = key = classKey - self._classes_brace_level[classKey] = self.braceDepth + key = classKey + + self.curClass = newClass if not key.endswith("::") and not key.endswith(" ") and len(key) != 0: if key in self.classes: @@ -2583,20 +2649,15 @@ def _evaluate_class_stack(self): newClass.show() assert key not in self.classes # namespace collision self.classes[key] = newClass - global parseHistory - parseHistory.append( - {"braceDepth": self.braceDepth, "item_type": "class", "item": newClass} - ) def evalute_forward_decl(self): trace_print("FORWARD DECL", self.nameStack) assert self.nameStack[0] in ("class", "struct") name = self.nameStack[-1] if self.curClass: - klass = self.classes[self.curClass] - klass["forward_declares"][self.curAccessSpecifier].append(name) + self.curClass["forward_declares"][self.curAccessSpecifier].append(name) if self.curAccessSpecifier == "public": - klass._public_forward_declares.append(name) + self.curClass._public_forward_declares.append(name) else: self._forward_decls.append(name) @@ -2670,7 +2731,7 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): headerFileStr = headerFileName else: raise Exception("Arg type must be either file or string") - self.curClass = "" + self.curClass = None # nested classes have parent::nested, but no extra namespace, # this keeps the API compatible, TODO proper namespace for everything. @@ -2725,12 +2786,13 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): "curAccessSpecifier changed/defaulted to %s", self.curAccessSpecifier ) self.initextra() - # Old namestacks for a given level - self.nameStackHistory = [] + self.anon_struct_counter = 0 self.anon_union_counter = 0 self.anon_class_counter = 0 + self.typedef_encountered = False + #: Using directives in this header outside of class scope: key is #: full name for lookup, value is :class:`.CppVariable` self.using = {} @@ -2881,7 +2943,7 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): continue if parenDepth == 0 and tok.type == "{": - if len(self.nameStack) >= 2 and is_namespace( + if nslen >= 2 and is_namespace( self.nameStack ): # namespace {} with no name used in boost, this sets default? if ( @@ -2942,49 +3004,17 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): self._evaluate_stack() self.braceDepth -= 1 - # if self.curClass: - # debug_print( - # "CURBD %s", self._classes_brace_level[self.curClass] - # ) - - if (self.braceDepth == 0) or ( - self.curClass - and self._classes_brace_level[self.curClass] == self.braceDepth - ): + if self.braceDepth == 0 or self.curClass: trace_print("END OF CLASS DEF") if self.accessSpecifierStack: self.curAccessSpecifier = self.accessSpecifierStack[-1] self.accessSpecifierStack = self.accessSpecifierStack[:-1] - if self.curClass and self.classes[self.curClass]["parent"]: - thisClass = self.classes[self.curClass] - self.curClass = self.curClass[ - : -(len(thisClass["name"]) + 2) - ] - # Detect anonymous union members - if ( - self.curClass - and thisClass["declaration_method"] == "union" - and thisClass["name"].startswith("<") - and self.lex.token_if(";") - ): - debug_print("Creating anonymous union") - # Force the processing of an anonymous union - self.nameStack = [""] - self.stack = self.nameStack + [";"] - debug_print("pre eval anon stack") - self._evaluate_stack(";") - debug_print("post eval anon stack") - self.stack = [] - self.nameStack = [] - self.stmtTokens = [] - else: - self.curClass = "" - self.stack = [] - self.stmtTokens = [] + if self.curClass: + self._finish_class_def() + elif tok.type in _namestack_append_tokens: self.nameStack.append(tok.value) - nameStackAppended = True elif tok.type in _namestack_pass_tokens: pass elif tok.type in _namestack_str_tokens: @@ -3024,10 +3054,8 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): elif tok.type == "(": parenDepth += 1 self.nameStack.append(tok.value) - nameStackAppended = True elif tok.type == ")": self.nameStack.append(tok.value) - nameStackAppended = True if parenDepth != 0: parenDepth -= 1 @@ -3064,8 +3092,7 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): ) self.finalize() - global parseHistory - parseHistory = [] + # Delete some temporary variables for key in [ "_precomp_macro_buf", @@ -3076,11 +3103,10 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): "nameSpaces", "curAccessSpecifier", "accessSpecifierStack", - "nameStackHistory", "anon_struct_counter", "anon_union_counter", "anon_class_counter", - "_classes_brace_level", + "typedef_encountered", "_forward_decls", "stack", "mainClass", @@ -3089,6 +3115,7 @@ def __init__(self, headerFileName, argType="file", encoding=None, **kwargs): "stmtTokens", "typedefs_order", "curTemplate", + "curClass", ]: del self.__dict__[key] @@ -3128,6 +3155,16 @@ def _next_token_must_be(self, *tokenTypes): raise self._parse_error((tok,), "' or '".join(tokenTypes)) return tok + def _consume_up_to(self, rtoks, *token_types): + token = self.lex.token + while True: + tok = token() + rtoks.append(tok) + if tok.type in token_types: + break + + return rtoks + _end_balanced_tokens = {">", "}", "]", ")", "DBL_RBRACKET"} _balanced_token_map = { "<": ">", @@ -3241,8 +3278,6 @@ def _discard_ctor_initializer(self): def _evaluate_stack(self, token=None): """Evaluates the current name stack""" - nameStackCopy = self.nameStack[:] - debug_print( "Evaluating stack %s\n BraceDepth: %s (called from %s)", self.nameStack, @@ -3250,17 +3285,10 @@ def _evaluate_stack(self, token=None): debug_caller_lineno, ) - # Handle special case of overloading operator () - if "operator()(" in "".join(self.nameStack): - operator_index = self.nameStack.index("operator") - self.nameStack.pop(operator_index + 2) - self.nameStack.pop(operator_index + 1) - self.nameStack[operator_index] = "operator()" - - if len(self.curClass): - debug_print("%s (%s) ", self.curClass, self.curAccessSpecifier) - else: - debug_print(" (%s) ", self.curAccessSpecifier) + # if len(self.curClass): + # debug_print("%s (%s) ", self.curClass, self.curAccessSpecifier) + # else: + # debug_print(" (%s) ", self.curAccessSpecifier) # Filter special case of array with casting in it try: @@ -3275,17 +3303,21 @@ def _evaluate_stack(self, token=None): except: pass + if len(self.nameStack) == 0: + debug_print("trace (Empty Stack)") + return + # if 'typedef' in self.nameStack: self._evaluate_typedef() # allows nested typedefs, probably a bad idea - if ( + elif ( not self.curClass - and "typedef" in self.nameStack + and self.nameStack[0] == "typedef" and ( - ( + self.stack[-1] == ";" + or ( "struct" not in self.nameStack and "union" not in self.nameStack and "enum" not in self.nameStack ) - or self.stack[-1] == ";" ) ): debug_print("trace") @@ -3293,9 +3325,6 @@ def _evaluate_stack(self, token=None): self._evaluate_typedef() return - elif len(self.nameStack) == 0: - debug_print("trace (Empty Stack)") - return elif self.nameStack[0] == "namespace": # Taken care of outside of here pass @@ -3344,13 +3373,13 @@ def _evaluate_stack(self, token=None): atype["raw_type"] = ns + atype["type"] if self.curClass: - klass = self.classes[self.curClass] + klass = self.curClass klass["using"][alias] = atype else: # lookup is done alias = self.current_namespace() + alias self.using[alias] = atype - elif is_method_namestack(self.stack) and "(" in self.nameStack: + elif "(" in self.nameStack and is_method_namestack(self.stack): debug_print("trace") self._evaluate_method_stack() elif is_enum_namestack(self.nameStack): @@ -3358,26 +3387,8 @@ def _evaluate_stack(self, token=None): self._parse_enum() self.stack = [] self.stmtTokens = [] - elif ( - len(self.nameStack) == 1 - and len(self.nameStackHistory) > self.braceDepth - and ( - self.nameStackHistory[self.braceDepth][0][0:2] == ["typedef", "struct"] - or self.nameStackHistory[self.braceDepth][0][0:2] - == ["typedef", "union"] - ) - ): - # Look for the name of a typedef struct: struct typedef {...] StructName; or unions to get renamed - debug_print("found the naming of a union") - type_name_to_rename = self.nameStackHistory[self.braceDepth][1] - new_name = self.nameStack[0] - type_to_rename = self.classes[type_name_to_rename] - type_to_rename["name"] = self.nameStack[0] - # Now re install it in its new location - self.classes[new_name] = type_to_rename - if new_name != type_name_to_rename: - del self.classes[type_name_to_rename] - elif is_property_namestack(self.nameStack) and self.stack[-1] == ";": + + elif self.stack[-1] == ";" and is_property_namestack(self.nameStack): debug_print("trace") if self.nameStack[0] in ("class", "struct") and len(self.stack) == 3: self.evalute_forward_decl() @@ -3388,13 +3399,16 @@ def _evaluate_stack(self, token=None): else: self._evaluate_property_stack() # catches class props and structs in a namespace - elif ( - self.nameStack[0] in ("class", "struct", "union") - or self.nameStack[0] == "typedef" - and self.nameStack[1] in ("struct", "union") + elif self.nameStack[0] in ("class", "struct", "union"): + debug_print("trace") + self._evaluate_class_stack() + + elif self.nameStack[0] == "typedef" and self.nameStack[1] in ( + "struct", + "union", ): - # Parsing a union can reuse much of the class parsing debug_print("trace") + self.typedef_encountered = True self._evaluate_class_stack() elif not self.curClass: @@ -3408,11 +3422,6 @@ def _evaluate_stack(self, token=None): else: debug_print("Discarded statement %s", self.nameStack) - try: - self.nameStackHistory[self.braceDepth] = (nameStackCopy, self.curClass) - except: - self.nameStackHistory.append((nameStackCopy, self.curClass)) - # its a little confusing to have some if/else above return and others not, and then clearning the nameStack down here self.nameStack = [] self.lex.doxygenCommentCache = "" @@ -3567,12 +3576,11 @@ def _parse_enum(self): self._install_enum(newEnum, instancesData) def _install_enum(self, newEnum, instancesData): - if len(self.curClass): + if self.curClass: newEnum["namespace"] = self.cur_namespace(False) - klass = self.classes[self.curClass] - klass["enums"][self.curAccessSpecifier].append(newEnum) + self.curClass["enums"][self.curAccessSpecifier].append(newEnum) if self.curAccessSpecifier == "public" and "name" in newEnum: - klass._public_enums[newEnum["name"]] = newEnum + self.curClass._public_enums[newEnum["name"]] = newEnum else: newEnum["namespace"] = self.cur_namespace(True) self.enums.append(newEnum) @@ -3662,7 +3670,7 @@ def _strip_parent_keys(self): for k in obj.keys(): trace_print("-Try key", k) trace_print("-type", type(obj[k])) - if k in ["nameStackHistory", "parent", "_public_typedefs"]: + if k in ["parent", "_public_typedefs"]: continue if type(obj[k]) == list: for i in obj[k]: diff --git a/test/TestSampleClass.h b/test/TestSampleClass.h index 439635c..08ce500 100644 --- a/test/TestSampleClass.h +++ b/test/TestSampleClass.h @@ -307,7 +307,7 @@ extern "C" { class ExternClass { ExternClass(); - } + }; }; // Bug 3514671 @@ -502,14 +502,14 @@ class BlueJay : public Bird, public virtual Food class Pea : public Vegetable { int i; -} +}; // Bug 3567172 class Pear { enum Stem stem_property; -} +}; // Bug 3567854 and 3568241 struct Beans @@ -572,7 +572,7 @@ class ClassAfterMagicMacro { public: ClassAfterMagicMacro(); -} +}; // Bug BitBucket #4 typedef unsigned int uint; @@ -583,7 +583,7 @@ typedef std::map StrStrMap; class AfterTypedefClass { public: -} +}; // Bug BitBucket #5 class Herb @@ -633,7 +633,7 @@ class Plumb class Peach * Plumb::myMethod( class Peach * pInPtr ) { return pInPtr; -} +}; // Bug BitBucket #9 class Grape @@ -667,7 +667,7 @@ class Hen public: void add(int a=100, b=0xfd, float c=1.7e-3, float d=3.14); void join(string s1="", string s2="nothing"); -} +}; // Bug BitBucket #19 template