diff --git a/parser.py b/parser.py index 8035758..da35c24 100644 --- a/parser.py +++ b/parser.py @@ -3,6 +3,7 @@ Contains class PARSER ''' from itertools import count import os +from tokenizer import TOKENIZER #Element type definitions. Used in the parse process. ELEMENT_TYPE_PREPROCESS = 1 @@ -82,6 +83,7 @@ class PARSEOBJECT: _passes = count(0) def __init__(self): + self.tokenize = TOKENIZER() self.parseline = [] self.parsefile = [] self.passes = 0 diff --git a/tokenizer.py b/tokenizer.py new file mode 100644 index 0000000..8a06552 --- /dev/null +++ b/tokenizer.py @@ -0,0 +1,70 @@ +''' +Contains class TOKENIZER +''' +from contextlib import ContextDecorator +from itertools import count + +class TOKENIZEOBJECT: + _passes = count(0) + _analyzed = [] + + def __exit__(self, *exc): + return False + + def __init__(self): + self.passes = 0 + self.analyzeline = [] + self.analyzed = [] + + class Decorators(ContextDecorator): + @classmethod + def typedef_struct(ContextDecorator): + TOKENIZER._analyzed.append('TOKEN_TYPEDEF_STRUCT') + return + + @classmethod + def typedef_enum(ContextDecorator): + TOKENIZER._analyzed.append('TOKEN_TYPEDEF_ENUM') + return + + @classmethod + def enum(ContextDecorator): + TOKENIZER._analyzed.append('TOKEN_ENUM') + return + + @classmethod + def tagname(ContextDecorator): + TOKENIZER._analyzed.append('TOKEN_TAG_NAME') + return + + @classmethod + def alias(ContextDecorator): + TOKENIZER._analyzed.append('TOKEN_ALIAS') + return + + @Decorators.typedef_struct + def TD_struct(self, tsl): + ''' + Takes a Typedef Struct 'list' and appends all items (i) in that list to _analyzed. + ''' + for i in tsl: + self._analyzed.append(i) + return + + @Decorators.tagname + def TD_tagname(self, tn): + ''' + Takes a Typedef tagname and appends it to _analyzed. + ''' + self._analyzed.append(tn) + return + +class TOKENIZER(TOKENIZEOBJECT): + _ids = count(0) + _passes = count(0) + + def __init__(self): + self.id = next(self._ids) + self.tupline = [] + self.tupfile = [] + self.passes = next(self._passes) \ No newline at end of file