diff --git a/etgtools/extractors.py b/etgtools/extractors.py index f452b9b9..a87e462b 100644 --- a/etgtools/extractors.py +++ b/etgtools/extractors.py @@ -15,12 +15,13 @@ wxWidgets API info which we need from them. import sys import os import pprint -import xml.etree.ElementTree as ET +from typing import Optional +import xml.etree.ElementTree as et import copy -from .tweaker_tools import FixWxPrefix, magicMethods, \ +from .tweaker_tools import FixWxPrefix, MethodType, magicMethods, \ guessTypeInt, guessTypeFloat, guessTypeStr, \ - textfile_open + textfile_open, Signature from sphinxtools.utilities import findDescendants if sys.version_info >= (3, 11): @@ -285,6 +286,7 @@ class FunctionDef(BaseDef, FixWxPrefix): self.type = None self.definition = '' self.argsString = '' + self.signature: Optional[Signature] = None self.pyArgsString = '' self.isOverloaded = False self.overloads = [] @@ -406,7 +408,10 @@ class FunctionDef(BaseDef, FixWxPrefix): else: parent = self.klass item = self.findOverload(matchText) + assert item is not None item.pyName = newName + if item.signature: + item.signature.method_name = newName item.__dict__.update(kw) if item is self and not self.hasOverloads(): @@ -470,8 +475,8 @@ class FunctionDef(BaseDef, FixWxPrefix): Create a pythonized version of the argsString in function and method items that can be used as part of the docstring. """ - params = list() - returns = list() + params: list[Signature.Parameter] = [] + returns: list[str] = [] if self.type and self.type != 'void': returns.append(self.cleanType(self.type)) @@ -483,6 +488,7 @@ class FunctionDef(BaseDef, FixWxPrefix): 'wxArrayInt()' : '[]', 'wxEmptyString': "''", # Makes signatures much shorter } + P = Signature.Parameter if isinstance(self, CppMethodDef): # rip apart the argsString instead of using the (empty) list of parameters lastP = self.argsString.rfind(')') @@ -495,22 +501,15 @@ class FunctionDef(BaseDef, FixWxPrefix): if '=' in arg: default = arg.split('=')[1].strip() arg = arg.split('=')[0].strip() - if default in defValueMap: - default = defValueMap.get(default) - else: - default = self.fixWxPrefix(default, True) + default = defValueMap.get(default, default) + default = self.fixWxPrefix(default, True) # now grab just the last word, it should be the variable name # The rest will be the type information arg_type, arg = arg.rsplit(None, 1) arg, arg_type = self.parseNameAndType(arg, arg_type) - if arg_type: - if default == 'None': - arg = f'{arg}: Optional[{arg_type}]' - else: - arg = f'{arg}: {arg_type}' - if default: - arg += '=' + default - params.append(arg) + params.append(P(arg, arg_type, default)) + if default == 'None': + params[-1].make_optional() else: for param in self.items: assert isinstance(param, ParamDef) @@ -523,35 +522,35 @@ class FunctionDef(BaseDef, FixWxPrefix): if param_type: returns.append(param_type) else: + default = '' if param.inOut: if param_type: returns.append(param_type) if param.default: default = param.default - if default in defValueMap: - default = defValueMap.get(default) - if param_type: - if default == 'None': - s = f'{s}: Optional[{param_type}]' - else: - s = f'{s}: {param_type}' + default = defValueMap.get(default, default) default = '|'.join([self.cleanName(x, True) for x in default.split('|')]) - s = f'{s}={default}' - elif param_type: - s = f'{s}: {param_type}' - params.append(s) - - self.pyArgsString = f"({', '.join(params)})" + params.append(P(s, param_type, default)) + if default == 'None': + params[-1].make_optional() + if getattr(self, 'isCtor', False): + name = '__init__' + else: + name = self.name or self.pyName + name = self.fixWxPrefix(name) # __bool__ and __nonzero__ need to be defined as returning int for SIP, but for Python # __bool__ is required to return a bool: - if (self.name or self.pyName) in ('__bool__', '__nonzero__'): - returns = ['bool'] - if not returns: - self.pyArgsString = f'{self.pyArgsString} -> None' + if name in ('__bool__', '__nonzero__'): + return_type = 'bool' + elif not returns: + return_type = 'None' elif len(returns) == 1: - self.pyArgsString = f'{self.pyArgsString} -> {returns[0]}' - elif len(returns) > 1: - self.pyArgsString = f"{self.pyArgsString} -> Tuple[{', '.join(returns)}]" + return_type = returns[0] + else: + return_type = f"Tuple[{', '.join(returns)}]" + kind = MethodType.STATIC_METHOD if getattr(self, 'isStatic', False) else MethodType.METHOD + self.signature = Signature(name, *params, return_type=return_type, method_type=kind) + self.pyArgsString = self.signature.args_string(False) def collectPySignatures(self): @@ -1283,7 +1282,7 @@ class CppMethodDef(MethodDef): NOTE: This one is not automatically extracted, but can be added to classes in the tweaker stage """ - def __init__(self, type, name, argsString, body, doc=None, isConst=False, + def __init__(self, type, name, argsString: str, body, doc=None, isConst=False, cppSignature=None, virtualCatcherCode=None, **kw): super(CppMethodDef, self).__init__() self.type = type diff --git a/etgtools/pi_generator.py b/etgtools/pi_generator.py index 219ec029..5cca2c52 100644 --- a/etgtools/pi_generator.py +++ b/etgtools/pi_generator.py @@ -413,16 +413,11 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix): stream.write('\n@overload') elif is_overload: stream.write('\n@overload') - stream.write('\ndef %s' % function.pyName) - argsString = function.pyArgsString - if not argsString: - argsString = '()' - if '(' != argsString[0]: - pos = argsString.find('(') - argsString = argsString[pos:] - argsString = argsString.replace('::', '.') - stream.write(argsString) - stream.write(':\n') + if not function.signature: + function.makePyArgsString() + assert function.signature is not None + for line in function.signature.definition_lines(): + stream.write(f'\n{line}') if is_overload: stream.write(' ...\n') else: @@ -572,19 +567,12 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix): value_type = '' if prop.getter: getter = self.find_method(klass, prop.getter) - if getter and '->' in getter.pyArgsString: - value_type = getter.pyArgsString.split('->')[1].strip() + if getter and getter.signature: + value_type = getter.signature.return_type if prop.setter: setter = self.find_method(klass, prop.setter) - if setter: - args = setter.pyArgsString.split('->')[0] - args = args.strip().strip('()') - args = args.split(',') - if args: - value_arg = args[0] - if ':' in value_arg: - value_type = value_arg.split(':')[1].strip() - value_type = value_type.split('=')[0] + if setter and setter.signature: + value_type = setter.signature[0].type_hint if prop.setter and prop.getter: if value_type: stream.write(f'{indent}@property\n') @@ -626,10 +614,6 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix): if method.isDtor: return - name = name or method.pyName or method.name - if name in magicMethods: - name = magicMethods[name] - # write the method declaration if not is_overload and method.hasOverloads(): for m in method.overloads: @@ -637,32 +621,16 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix): stream.write(f'\n{indent}@overload') elif is_overload: stream.write(f'\n{indent}@overload') - if method.isStatic: - stream.write('\n%s@staticmethod' % indent) - stream.write('\n%sdef %s' % (indent, name)) - argsString = method.pyArgsString - if not argsString: - argsString = '()' - if '(' != argsString[0]: - pos = argsString.find('(') - argsString = argsString[pos:] - if not method.isStatic: - if argsString == '()': - argsString = '(self)' - else: - argsString = '(self, ' + argsString[1:] - argsString = argsString.replace('::', '.') - if is_top_level_init: - # For classes derived from TopLevelWindow, the parent argument is allowed - # to be None. If the argsString isn't already marked this way due to - # having `=None` in it, add the Optional[type] typehint to it. - argsString = re.sub( - r'([,(]\s*)parent\s*:\s*((?!Optional\[)(\w[\w\d_\.]*))\s*([,)=])', - r'\g<1>parent: Optional[\g<3>]\g<4>', - argsString, - ) - stream.write(argsString) - stream.write(':\n') + if not method.signature: + method.makePyArgsString() + assert method.signature is not None + if name is not None: + method.signature.method_name = name + if is_top_level_init and 'parent' in method.signature: + method.signature['parent'].make_optional() + for line in method.signature.definition_lines(): + stream.write(f'\n{indent}{line}') + stream.write('\n') indent2 = indent + ' '*4 # docstring diff --git a/etgtools/sphinx_generator.py b/etgtools/sphinx_generator.py index b1b333f5..4ed8087d 100644 --- a/etgtools/sphinx_generator.py +++ b/etgtools/sphinx_generator.py @@ -14,6 +14,7 @@ the various XML elements passed by the Phoenix extractors into ReST format. """ # Standard library stuff +import keyword import os import operator import sys @@ -28,7 +29,7 @@ import xml.etree.ElementTree as ET import etgtools.extractors as extractors import etgtools.generators as generators from etgtools.item_module_map import ItemModuleMap -from etgtools.tweaker_tools import removeWxPrefix +from etgtools.tweaker_tools import removeWxPrefix, ParameterType # Sphinx-Phoenix specific stuff from sphinxtools.inheritance import InheritanceDiagram @@ -580,31 +581,14 @@ class ParameterList(Node): if xml_item.hasOverloads() and not is_overload: return - arguments = xml_item.pyArgsString + if xml_item.signature is None: + xml_item.makePyArgsString() + assert xml_item.signature is not None + signature = xml_item.signature.signature() + arguments = list(xml_item.signature) if not arguments: return - if hasattr(xml_item, 'isStatic') and not xml_item.isStatic: - if arguments[:2] == '()': - return - - arguments = arguments[1:] - - if '->' in arguments: - arguments, dummy = arguments.split("->") - - arguments = arguments.strip() - if arguments.endswith(','): - arguments = arguments[0:-1] - - if arguments.startswith('('): - arguments = arguments[1:] - if arguments.endswith(')'): - arguments = arguments[0:-1] - - signature = name + '(%s)'%arguments - arguments = arguments.split(',') - py_parameters = [] for key, parameter in self.py_parameters.items(): pdef = parameter.pdef @@ -619,39 +603,23 @@ class ParameterList(Node): ' ==> Parameter list from wxWidgets XML items: %s\n\n' \ 'This may be a documentation bug in wxWidgets or a side-effect of removing the `wx` prefix from signatures.\n\n' - theargs = [] - for arg in arguments: - arg = arg.split(':')[0].strip() # Remove the typehint - if arg in ('_from', '_def', '_is'): # Reserved Python keywords we've had to rename - arg = arg[1:] - myarg = arg.split('=')[0].strip() - if myarg: - theargs.append(myarg) - - if '*' in arg or ')' in arg: + arg_name = arg.name + if arg.position_type in (ParameterType.VAR_ARGS, ParameterType.KWARGS): continue + if arg_name.startswith('_') and keyword.iskeyword(arg_name[1:]): # Reserved Python keywords we've had to rename + arg_name = arg_name[1:] + + #if '*' in arg_name: + # continue - arg = arg.split('=')[0].strip() - - if arg and arg not in py_parameters: - + if arg_name not in py_parameters: class_name = '' if hasattr(xml_item, 'className') and xml_item.className is not None: class_name = wx2Sphinx(xml_item.className)[1] + '.' print((message % (class_name + name, arg, signature, py_parameters))) -## for param in py_parameters: -## if param not in theargs: -## class_name = '' -## if hasattr(xml_item, 'className') and xml_item.className is not None: -## class_name = wx2Sphinx(xml_item.className)[1] + '.' -## -## print '\n ||| %s;%s;%s |||\n'%(class_name[0:-1], signature, param) -## with open('mismatched.txt', 'a') as fid: -## fid.write('%s;%s;%s\n'%(class_name[0:-1], signature, param)) - # ----------------------------------------------------------------------- diff --git a/etgtools/tweaker_tools.py b/etgtools/tweaker_tools.py index ff52b1b7..0e6f38fe 100644 --- a/etgtools/tweaker_tools.py +++ b/etgtools/tweaker_tools.py @@ -12,6 +12,7 @@ Some helpers and utility functions that can assist with the tweaker stage of the ETG scripts. """ +import enum import etgtools as extractors from .generators import textfile_open import keyword @@ -19,7 +20,7 @@ import re import sys, os import copy import textwrap -from typing import Optional, Tuple +from typing import NamedTuple, Optional, Tuple, Union isWindows = sys.platform.startswith('win') @@ -40,6 +41,174 @@ magicMethods = { } +class AutoConversionInfo(NamedTuple): + convertables: tuple[str, ...] # String type-hints for each of the types that can be automatically converted to this class + code: str # Code that will be added to SIP for this conversion + + +class ParameterType(enum.Enum): + VAR_ARGS = enum.auto() + KWARGS = enum.auto() + POSITIONAL_ONLY = enum.auto() + DEFAULT = enum.auto() + + +class MethodType(enum.Enum): + STATIC_METHOD = enum.auto() # Class @staticmethod method + CLASS_METHOD = enum.auto() # Class @classmethod method + METHOD = enum.auto() # Class regular method + FUNCTION = enum.auto() # non-class function + + +class Signature: + """Like inspect.Signature, but a bit simpler because we need it only for a few purposes: + - Creation from a C++ args string + - We *don't* want stringized (ie: all of them) type-hints to be evaluated, since we're + processing them in a context where most of them will be unresolvable. + """ + + class Parameter: + __slots__ = ('name', 'type_hint', 'default', 'position_type', ) + name: str + type_hint: Optional[str] + default: Optional[str] + position_type: ParameterType + + def __init__(self, name: str, type_hint: Optional[str] = None, default: Optional[str] = None, position_type: ParameterType = ParameterType.DEFAULT) -> None: + if name.startswith('**'): + name = name[2:] + position_type = ParameterType.KWARGS + elif name.startswith('*'): + name = name[1:] + position_type = ParameterType.VAR_ARGS + type_hint = type_hint.replace('::', '.') if type_hint else None + default = default.replace('::', '.') if default else None + self.name = name + self.type_hint = type_hint + self.default = default + self.position_type = position_type + + @property + def _position_marking(self) -> str: + if self.position_type is ParameterType.KWARGS: + return '**' + elif self.position_type is ParameterType.VAR_ARGS: + return '*' + return '' + + def untyped(self) -> str: + if self.default is None: + return f'{self._position_marking}{self.name}' + else: + return f'{self._position_marking}{self.name}={self.default}' + + def __str__(self) -> str: + if self.type_hint is None and self.default is None: + return f'{self._position_marking}{self.name}' + elif self.type_hint is None: + return f'{self._position_marking}{self.name}={self.default}' + elif self.default is None: + return f'{self._position_marking}{self.name}: {self.type_hint}' + else: + return f'{self._position_marking}{self.name}: {self.type_hint}={self.default}' + + def make_optional(self) -> None: + if self.type_hint is not None and not self.type_hint.startswith('Optional['): + self.type_hint = f'Optional[{self.type_hint}]' + + __slots__ = ('_method_name', 'return_type', '_parameters', '_method_type', ) + _method_name: str + return_type: Optional[str] + _parameters: dict[str, Parameter] + _method_type: MethodType + + def __init__(self, method_name: str, *parameters: Parameter, return_type: Optional[str] = None, method_type: MethodType = MethodType.METHOD) -> None: + self._parameters = { + p.name: p + for p in parameters + } + self.return_type = return_type.replace('::', '.') if return_type else None + self._method_type = method_type + self.method_name = method_name + + @property + def method_name(self) -> str: + return self._method_name + + @method_name.setter + def method_name(self, value: str, /) -> None: + self._method_name = magicMethods.get(value, value) + + def __getitem__(self, key: Union[str, int]) -> Parameter: + """Get parameter by name or by index. Indexing is into the paramters skips 'cls' and 'self' for + classmethods and methods. + """ + if isinstance(key, int): + key = list(self._parameters)[key] + if isinstance(key, str): + return self._parameters[key] + else: + raise TypeError(f'Indexing must be via parameter name or index, got {key}') + + def __iter__(self): + return iter(self._parameters.values()) + + def __contains__(self, parameter_name: str) -> bool: + return parameter_name in self._parameters + + def args_string(self, typed: bool = True, include_selfcls: bool = False) -> str: + """Get a string of just the parameters needed for the method signature, + optionally with 'self' or 'cls' where applicable, and type-hints + """ + if include_selfcls and self.is_classmethod: + parameters = (type(self).Parameter('cls'), *self._parameters.values()) + elif include_selfcls and self.is_method: + parameters = (type(self).Parameter('self'), *self._parameters.values()) + else: + parameters = self._parameters.values() + stringizer = str if typed else type(self).Parameter.untyped + return_type = f' -> {self.return_type}' if self.return_type else '' + return f'({', '.join(map(stringizer, parameters))}){return_type}' + + def signature(self, typed: bool = True) -> str: + """Get the full signature for the function/method, including method and + all required python syntax, optionally including all type-hints. + """ + return f'def {self.method_name}{self.args_string(typed, True)}:' + + def __str__(self) -> str: + return self.signature() + + def definition_lines(self, typed: bool = True) -> list[str]: + """return the lines required to write the full method definition, + including decorators + """ + if self.is_staticmethod: + lines = ['@staticmethod'] + elif self.is_classmethod: + lines = ['@classmethod'] + else: + lines = [] + lines.append(self.signature(typed)) + return lines + + @property + def is_staticmethod(self) -> bool: + return self._method_type is MethodType.STATIC_METHOD + + @property + def is_classmethod(self) -> bool: + return self._method_type is MethodType.CLASS_METHOD + + @property + def is_method(self) -> bool: + return self._method_type is MethodType.METHOD + + @property + def is_function(self) -> bool: + return self._method_type is MethodType.FUNCTION + + def removeWxPrefixes(node): """ Rename items with a 'wx' prefix to not have the prefix. If the back-end @@ -151,7 +320,10 @@ class FixWxPrefix(object): Finally, the 'wx.' prefix is added if needed. """ name = re.sub(r'(const(?![\w\d]))', '', name) # remove 'const', but not 'const'raints - for txt in ['*', '&', ' ']: + replacements = [' ', '*'] + if not is_expression: + replacements.extend(['&']) + for txt in replacements: name = name.replace(txt, '') name = name.replace('::', '.') if not is_expression: