reworks how typed-argstrings are handled

Pulls out the data into a separate class, `tweaker_tools.Signature`.
This simplifies stringizing the args string, and allows for some
checks to be written in a much less complex way. The motivator for
this was adding unions to type-hints (for the upcoming automatically
converted python->wx types). This made parsing the already stringized
args string very complex to be able to handle the potential for commas
in `Union`.
This commit is contained in:
lojack5
2025-01-16 22:15:13 -07:00
parent e76f298bf1
commit 03d7f1e8c9
4 changed files with 245 additions and 138 deletions

View File

@@ -15,12 +15,13 @@ wxWidgets API info which we need from them.
import sys
import os
import pprint
import xml.etree.ElementTree as ET
from typing import Optional
import xml.etree.ElementTree as et
import copy
from .tweaker_tools import FixWxPrefix, magicMethods, \
from .tweaker_tools import FixWxPrefix, MethodType, magicMethods, \
guessTypeInt, guessTypeFloat, guessTypeStr, \
textfile_open
textfile_open, Signature
from sphinxtools.utilities import findDescendants
if sys.version_info >= (3, 11):
@@ -285,6 +286,7 @@ class FunctionDef(BaseDef, FixWxPrefix):
self.type = None
self.definition = ''
self.argsString = ''
self.signature: Optional[Signature] = None
self.pyArgsString = ''
self.isOverloaded = False
self.overloads = []
@@ -406,7 +408,10 @@ class FunctionDef(BaseDef, FixWxPrefix):
else:
parent = self.klass
item = self.findOverload(matchText)
assert item is not None
item.pyName = newName
if item.signature:
item.signature.method_name = newName
item.__dict__.update(kw)
if item is self and not self.hasOverloads():
@@ -470,8 +475,8 @@ class FunctionDef(BaseDef, FixWxPrefix):
Create a pythonized version of the argsString in function and method
items that can be used as part of the docstring.
"""
params = list()
returns = list()
params: list[Signature.Parameter] = []
returns: list[str] = []
if self.type and self.type != 'void':
returns.append(self.cleanType(self.type))
@@ -483,6 +488,7 @@ class FunctionDef(BaseDef, FixWxPrefix):
'wxArrayInt()' : '[]',
'wxEmptyString': "''", # Makes signatures much shorter
}
P = Signature.Parameter
if isinstance(self, CppMethodDef):
# rip apart the argsString instead of using the (empty) list of parameters
lastP = self.argsString.rfind(')')
@@ -495,22 +501,15 @@ class FunctionDef(BaseDef, FixWxPrefix):
if '=' in arg:
default = arg.split('=')[1].strip()
arg = arg.split('=')[0].strip()
if default in defValueMap:
default = defValueMap.get(default)
else:
default = self.fixWxPrefix(default, True)
default = defValueMap.get(default, default)
default = self.fixWxPrefix(default, True)
# now grab just the last word, it should be the variable name
# The rest will be the type information
arg_type, arg = arg.rsplit(None, 1)
arg, arg_type = self.parseNameAndType(arg, arg_type)
if arg_type:
if default == 'None':
arg = f'{arg}: Optional[{arg_type}]'
else:
arg = f'{arg}: {arg_type}'
if default:
arg += '=' + default
params.append(arg)
params.append(P(arg, arg_type, default))
if default == 'None':
params[-1].make_optional()
else:
for param in self.items:
assert isinstance(param, ParamDef)
@@ -523,35 +522,35 @@ class FunctionDef(BaseDef, FixWxPrefix):
if param_type:
returns.append(param_type)
else:
default = ''
if param.inOut:
if param_type:
returns.append(param_type)
if param.default:
default = param.default
if default in defValueMap:
default = defValueMap.get(default)
if param_type:
if default == 'None':
s = f'{s}: Optional[{param_type}]'
else:
s = f'{s}: {param_type}'
default = defValueMap.get(default, default)
default = '|'.join([self.cleanName(x, True) for x in default.split('|')])
s = f'{s}={default}'
elif param_type:
s = f'{s}: {param_type}'
params.append(s)
self.pyArgsString = f"({', '.join(params)})"
params.append(P(s, param_type, default))
if default == 'None':
params[-1].make_optional()
if getattr(self, 'isCtor', False):
name = '__init__'
else:
name = self.name or self.pyName
name = self.fixWxPrefix(name)
# __bool__ and __nonzero__ need to be defined as returning int for SIP, but for Python
# __bool__ is required to return a bool:
if (self.name or self.pyName) in ('__bool__', '__nonzero__'):
returns = ['bool']
if not returns:
self.pyArgsString = f'{self.pyArgsString} -> None'
if name in ('__bool__', '__nonzero__'):
return_type = 'bool'
elif not returns:
return_type = 'None'
elif len(returns) == 1:
self.pyArgsString = f'{self.pyArgsString} -> {returns[0]}'
elif len(returns) > 1:
self.pyArgsString = f"{self.pyArgsString} -> Tuple[{', '.join(returns)}]"
return_type = returns[0]
else:
return_type = f"Tuple[{', '.join(returns)}]"
kind = MethodType.STATIC_METHOD if getattr(self, 'isStatic', False) else MethodType.METHOD
self.signature = Signature(name, *params, return_type=return_type, method_type=kind)
self.pyArgsString = self.signature.args_string(False)
def collectPySignatures(self):
@@ -1283,7 +1282,7 @@ class CppMethodDef(MethodDef):
NOTE: This one is not automatically extracted, but can be added to
classes in the tweaker stage
"""
def __init__(self, type, name, argsString, body, doc=None, isConst=False,
def __init__(self, type, name, argsString: str, body, doc=None, isConst=False,
cppSignature=None, virtualCatcherCode=None, **kw):
super(CppMethodDef, self).__init__()
self.type = type

View File

@@ -413,16 +413,11 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix):
stream.write('\n@overload')
elif is_overload:
stream.write('\n@overload')
stream.write('\ndef %s' % function.pyName)
argsString = function.pyArgsString
if not argsString:
argsString = '()'
if '(' != argsString[0]:
pos = argsString.find('(')
argsString = argsString[pos:]
argsString = argsString.replace('::', '.')
stream.write(argsString)
stream.write(':\n')
if not function.signature:
function.makePyArgsString()
assert function.signature is not None
for line in function.signature.definition_lines():
stream.write(f'\n{line}')
if is_overload:
stream.write(' ...\n')
else:
@@ -572,19 +567,12 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix):
value_type = ''
if prop.getter:
getter = self.find_method(klass, prop.getter)
if getter and '->' in getter.pyArgsString:
value_type = getter.pyArgsString.split('->')[1].strip()
if getter and getter.signature:
value_type = getter.signature.return_type
if prop.setter:
setter = self.find_method(klass, prop.setter)
if setter:
args = setter.pyArgsString.split('->')[0]
args = args.strip().strip('()')
args = args.split(',')
if args:
value_arg = args[0]
if ':' in value_arg:
value_type = value_arg.split(':')[1].strip()
value_type = value_type.split('=')[0]
if setter and setter.signature:
value_type = setter.signature[0].type_hint
if prop.setter and prop.getter:
if value_type:
stream.write(f'{indent}@property\n')
@@ -626,10 +614,6 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix):
if method.isDtor:
return
name = name or method.pyName or method.name
if name in magicMethods:
name = magicMethods[name]
# write the method declaration
if not is_overload and method.hasOverloads():
for m in method.overloads:
@@ -637,32 +621,16 @@ class PiWrapperGenerator(generators.WrapperGeneratorBase, FixWxPrefix):
stream.write(f'\n{indent}@overload')
elif is_overload:
stream.write(f'\n{indent}@overload')
if method.isStatic:
stream.write('\n%s@staticmethod' % indent)
stream.write('\n%sdef %s' % (indent, name))
argsString = method.pyArgsString
if not argsString:
argsString = '()'
if '(' != argsString[0]:
pos = argsString.find('(')
argsString = argsString[pos:]
if not method.isStatic:
if argsString == '()':
argsString = '(self)'
else:
argsString = '(self, ' + argsString[1:]
argsString = argsString.replace('::', '.')
if is_top_level_init:
# For classes derived from TopLevelWindow, the parent argument is allowed
# to be None. If the argsString isn't already marked this way due to
# having `=None` in it, add the Optional[type] typehint to it.
argsString = re.sub(
r'([,(]\s*)parent\s*:\s*((?!Optional\[)(\w[\w\d_\.]*))\s*([,)=])',
r'\g<1>parent: Optional[\g<3>]\g<4>',
argsString,
)
stream.write(argsString)
stream.write(':\n')
if not method.signature:
method.makePyArgsString()
assert method.signature is not None
if name is not None:
method.signature.method_name = name
if is_top_level_init and 'parent' in method.signature:
method.signature['parent'].make_optional()
for line in method.signature.definition_lines():
stream.write(f'\n{indent}{line}')
stream.write('\n')
indent2 = indent + ' '*4
# docstring

View File

@@ -14,6 +14,7 @@ the various XML elements passed by the Phoenix extractors into ReST format.
"""
# Standard library stuff
import keyword
import os
import operator
import sys
@@ -28,7 +29,7 @@ import xml.etree.ElementTree as ET
import etgtools.extractors as extractors
import etgtools.generators as generators
from etgtools.item_module_map import ItemModuleMap
from etgtools.tweaker_tools import removeWxPrefix
from etgtools.tweaker_tools import removeWxPrefix, ParameterType
# Sphinx-Phoenix specific stuff
from sphinxtools.inheritance import InheritanceDiagram
@@ -580,31 +581,14 @@ class ParameterList(Node):
if xml_item.hasOverloads() and not is_overload:
return
arguments = xml_item.pyArgsString
if xml_item.signature is None:
xml_item.makePyArgsString()
assert xml_item.signature is not None
signature = xml_item.signature.signature()
arguments = list(xml_item.signature)
if not arguments:
return
if hasattr(xml_item, 'isStatic') and not xml_item.isStatic:
if arguments[:2] == '()':
return
arguments = arguments[1:]
if '->' in arguments:
arguments, dummy = arguments.split("->")
arguments = arguments.strip()
if arguments.endswith(','):
arguments = arguments[0:-1]
if arguments.startswith('('):
arguments = arguments[1:]
if arguments.endswith(')'):
arguments = arguments[0:-1]
signature = name + '(%s)'%arguments
arguments = arguments.split(',')
py_parameters = []
for key, parameter in self.py_parameters.items():
pdef = parameter.pdef
@@ -619,39 +603,23 @@ class ParameterList(Node):
' ==> Parameter list from wxWidgets XML items: %s\n\n' \
'This may be a documentation bug in wxWidgets or a side-effect of removing the `wx` prefix from signatures.\n\n'
theargs = []
for arg in arguments:
arg = arg.split(':')[0].strip() # Remove the typehint
if arg in ('_from', '_def', '_is'): # Reserved Python keywords we've had to rename
arg = arg[1:]
myarg = arg.split('=')[0].strip()
if myarg:
theargs.append(myarg)
if '*' in arg or ')' in arg:
arg_name = arg.name
if arg.position_type in (ParameterType.VAR_ARGS, ParameterType.KWARGS):
continue
if arg_name.startswith('_') and keyword.iskeyword(arg_name[1:]): # Reserved Python keywords we've had to rename
arg_name = arg_name[1:]
#if '*' in arg_name:
# continue
arg = arg.split('=')[0].strip()
if arg and arg not in py_parameters:
if arg_name not in py_parameters:
class_name = ''
if hasattr(xml_item, 'className') and xml_item.className is not None:
class_name = wx2Sphinx(xml_item.className)[1] + '.'
print((message % (class_name + name, arg, signature, py_parameters)))
## for param in py_parameters:
## if param not in theargs:
## class_name = ''
## if hasattr(xml_item, 'className') and xml_item.className is not None:
## class_name = wx2Sphinx(xml_item.className)[1] + '.'
##
## print '\n ||| %s;%s;%s |||\n'%(class_name[0:-1], signature, param)
## with open('mismatched.txt', 'a') as fid:
## fid.write('%s;%s;%s\n'%(class_name[0:-1], signature, param))
# -----------------------------------------------------------------------

View File

@@ -12,6 +12,7 @@ Some helpers and utility functions that can assist with the tweaker
stage of the ETG scripts.
"""
import enum
import etgtools as extractors
from .generators import textfile_open
import keyword
@@ -19,7 +20,7 @@ import re
import sys, os
import copy
import textwrap
from typing import Optional, Tuple
from typing import NamedTuple, Optional, Tuple, Union
isWindows = sys.platform.startswith('win')
@@ -40,6 +41,174 @@ magicMethods = {
}
class AutoConversionInfo(NamedTuple):
convertables: tuple[str, ...] # String type-hints for each of the types that can be automatically converted to this class
code: str # Code that will be added to SIP for this conversion
class ParameterType(enum.Enum):
VAR_ARGS = enum.auto()
KWARGS = enum.auto()
POSITIONAL_ONLY = enum.auto()
DEFAULT = enum.auto()
class MethodType(enum.Enum):
STATIC_METHOD = enum.auto() # Class @staticmethod method
CLASS_METHOD = enum.auto() # Class @classmethod method
METHOD = enum.auto() # Class regular method
FUNCTION = enum.auto() # non-class function
class Signature:
"""Like inspect.Signature, but a bit simpler because we need it only for a few purposes:
- Creation from a C++ args string
- We *don't* want stringized (ie: all of them) type-hints to be evaluated, since we're
processing them in a context where most of them will be unresolvable.
"""
class Parameter:
__slots__ = ('name', 'type_hint', 'default', 'position_type', )
name: str
type_hint: Optional[str]
default: Optional[str]
position_type: ParameterType
def __init__(self, name: str, type_hint: Optional[str] = None, default: Optional[str] = None, position_type: ParameterType = ParameterType.DEFAULT) -> None:
if name.startswith('**'):
name = name[2:]
position_type = ParameterType.KWARGS
elif name.startswith('*'):
name = name[1:]
position_type = ParameterType.VAR_ARGS
type_hint = type_hint.replace('::', '.') if type_hint else None
default = default.replace('::', '.') if default else None
self.name = name
self.type_hint = type_hint
self.default = default
self.position_type = position_type
@property
def _position_marking(self) -> str:
if self.position_type is ParameterType.KWARGS:
return '**'
elif self.position_type is ParameterType.VAR_ARGS:
return '*'
return ''
def untyped(self) -> str:
if self.default is None:
return f'{self._position_marking}{self.name}'
else:
return f'{self._position_marking}{self.name}={self.default}'
def __str__(self) -> str:
if self.type_hint is None and self.default is None:
return f'{self._position_marking}{self.name}'
elif self.type_hint is None:
return f'{self._position_marking}{self.name}={self.default}'
elif self.default is None:
return f'{self._position_marking}{self.name}: {self.type_hint}'
else:
return f'{self._position_marking}{self.name}: {self.type_hint}={self.default}'
def make_optional(self) -> None:
if self.type_hint is not None and not self.type_hint.startswith('Optional['):
self.type_hint = f'Optional[{self.type_hint}]'
__slots__ = ('_method_name', 'return_type', '_parameters', '_method_type', )
_method_name: str
return_type: Optional[str]
_parameters: dict[str, Parameter]
_method_type: MethodType
def __init__(self, method_name: str, *parameters: Parameter, return_type: Optional[str] = None, method_type: MethodType = MethodType.METHOD) -> None:
self._parameters = {
p.name: p
for p in parameters
}
self.return_type = return_type.replace('::', '.') if return_type else None
self._method_type = method_type
self.method_name = method_name
@property
def method_name(self) -> str:
return self._method_name
@method_name.setter
def method_name(self, value: str, /) -> None:
self._method_name = magicMethods.get(value, value)
def __getitem__(self, key: Union[str, int]) -> Parameter:
"""Get parameter by name or by index. Indexing is into the paramters skips 'cls' and 'self' for
classmethods and methods.
"""
if isinstance(key, int):
key = list(self._parameters)[key]
if isinstance(key, str):
return self._parameters[key]
else:
raise TypeError(f'Indexing must be via parameter name or index, got {key}')
def __iter__(self):
return iter(self._parameters.values())
def __contains__(self, parameter_name: str) -> bool:
return parameter_name in self._parameters
def args_string(self, typed: bool = True, include_selfcls: bool = False) -> str:
"""Get a string of just the parameters needed for the method signature,
optionally with 'self' or 'cls' where applicable, and type-hints
"""
if include_selfcls and self.is_classmethod:
parameters = (type(self).Parameter('cls'), *self._parameters.values())
elif include_selfcls and self.is_method:
parameters = (type(self).Parameter('self'), *self._parameters.values())
else:
parameters = self._parameters.values()
stringizer = str if typed else type(self).Parameter.untyped
return_type = f' -> {self.return_type}' if self.return_type else ''
return f'({', '.join(map(stringizer, parameters))}){return_type}'
def signature(self, typed: bool = True) -> str:
"""Get the full signature for the function/method, including method and
all required python syntax, optionally including all type-hints.
"""
return f'def {self.method_name}{self.args_string(typed, True)}:'
def __str__(self) -> str:
return self.signature()
def definition_lines(self, typed: bool = True) -> list[str]:
"""return the lines required to write the full method definition,
including decorators
"""
if self.is_staticmethod:
lines = ['@staticmethod']
elif self.is_classmethod:
lines = ['@classmethod']
else:
lines = []
lines.append(self.signature(typed))
return lines
@property
def is_staticmethod(self) -> bool:
return self._method_type is MethodType.STATIC_METHOD
@property
def is_classmethod(self) -> bool:
return self._method_type is MethodType.CLASS_METHOD
@property
def is_method(self) -> bool:
return self._method_type is MethodType.METHOD
@property
def is_function(self) -> bool:
return self._method_type is MethodType.FUNCTION
def removeWxPrefixes(node):
"""
Rename items with a 'wx' prefix to not have the prefix. If the back-end
@@ -151,7 +320,10 @@ class FixWxPrefix(object):
Finally, the 'wx.' prefix is added if needed.
"""
name = re.sub(r'(const(?![\w\d]))', '', name) # remove 'const', but not 'const'raints
for txt in ['*', '&', ' ']:
replacements = [' ', '*']
if not is_expression:
replacements.extend(['&'])
for txt in replacements:
name = name.replace(txt, '')
name = name.replace('::', '.')
if not is_expression: