From e39fd68f1e2e45363d1b7789fc5a81bb005a09b2 Mon Sep 17 00:00:00 2001 From: Geo5 Date: Thu, 24 Jul 2025 04:09:16 +0200 Subject: [PATCH 1/6] Implement two-stage etg execution --- build.py | 12 ++++++++++++ etgtools/tweaker_tools.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 3761ccc9a..32f04724e 100755 --- a/build.py +++ b/build.py @@ -1144,6 +1144,12 @@ def cmd_etg(options, args): etgfiles.remove(core_file) etgfiles.insert(0, core_file) + # We need two loops: + # 1. Cache all type auto conversion rules which get created in etgtools/tweaker_tools.FixWxPrefix + # 2. Actually create the sip files + # This is needed because each etg file is run in its own python invocation, so + # the data is lost between them. + is_newer = {} for script in etgfiles: sipfile = etg2sip(script) deps = [script] @@ -1157,6 +1163,12 @@ def cmd_etg(options, args): # run the script only if any dependencies are newer if newer_group(deps, sipfile): + is_newer[script] = True + runcmd('"%s" %s %s' % (PYTHON, script, flags + " --only-cache-auto-conversions")) + else: + is_newer[script] = False + for script in etgfiles: + if is_newer[script]: runcmd('"%s" %s %s' % (PYTHON, script, flags)) diff --git a/etgtools/tweaker_tools.py b/etgtools/tweaker_tools.py index 58ca5f3af..cdbe4963c 100644 --- a/etgtools/tweaker_tools.py +++ b/etgtools/tweaker_tools.py @@ -246,6 +246,20 @@ def removeWxPrefix(name): return name +def load_auto_conversions(destFile=None): + """Load FixWxPrefix auto conversions from cache file if it exists.""" + import json + if not destFile: + from buildtools.config import Config + + cfg = Config(noWxConfig=True) + phoenixRoot = cfg.ROOT_DIR + destFile = os.path.join(phoenixRoot, 'sip/gen', '__auto_conversion_cache__.json') + # Need to merge existing data with current data + if os.path.isfile(destFile): + with open(destFile, 'r', encoding='utf-8') as f: + return json.load(f) + return {} class FixWxPrefix(object): """ @@ -254,7 +268,23 @@ class FixWxPrefix(object): """ _coreTopLevelNames = None - _auto_conversions: dict[str, Tuple[str, ...]] = {} + _auto_conversions: dict[str, Tuple[str, ...]] = load_auto_conversions() + + @classmethod + def cache_auto_conversions(cls, destFile=None): + """Save current auto conversions to a cache.""" + import json + + if not destFile: + from buildtools.config import Config + + cfg = Config(noWxConfig=True) + phoenixRoot = cfg.ROOT_DIR + destFile = os.path.join(phoenixRoot, 'sip/gen', '__auto_conversion_cache__.json') + # We overwrite the cache file, as we should have loaded the existing values when + # initializing the class. + with textfile_open(destFile, 'wt') as f: + json.dump(FixWxPrefix._auto_conversions, f) @classmethod def register_autoconversion(cls, class_name: str, convertables: Tuple[str, ...]) -> None: @@ -371,6 +401,7 @@ def cleanType(self, type_name: str, is_input: bool = False) -> str: 'time_t': 'int', 'size_t': 'int', 'Int32': 'int', + 'UInt32': 'int', 'long': long_type, 'unsignedlong': long_type, 'ulong': long_type, @@ -950,6 +981,9 @@ def runGenerators(module): checkForUnitTestModule(module) generators = list() + if '--only-cache-auto-conversions' in sys.argv: + FixWxPrefix.cache_auto_conversions() + return # Create the code generator selected from command line args generators.append(getWrapperGenerator()) From 564b1f5744984325a226d26306e9c563c870561b Mon Sep 17 00:00:00 2001 From: Geo5 Date: Thu, 24 Jul 2025 04:09:46 +0200 Subject: [PATCH 2/6] Improve property type hints --- etgtools/pi_generator.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/etgtools/pi_generator.py b/etgtools/pi_generator.py index 840115ea6..d7483134e 100644 --- a/etgtools/pi_generator.py +++ b/etgtools/pi_generator.py @@ -571,19 +571,45 @@ def generatePyProperty(self, klass, prop, stream, indent): def _generateProperty(self, klass: extractors.ClassDef, prop: Union[extractors.PyPropertyDef, extractors.PropertyDef], stream, indent: str): if prop.ignored or piIgnored(prop): return + # Track separate value and return types. + # This prevents us from emitting Union types on the getter, which do not make sense in practice, + # as exactly one type will be returned, even if multiple types are allowed by the setter. value_type = '' + return_type = '' if prop.getter: getter = self.find_method(klass, prop.getter) if getter and getter.signature: - value_type = getter.signature.return_type + return_type = getter.signature.return_type if prop.setter: setter = self.find_method(klass, prop.setter) - if setter and setter.signature: - value_type = setter.signature[0].type_hint + # The setter can be overloaded and we should find all setters which have exactly one argument and Union their types. + # We need to do this, because the property-setter only has one argument, so we assume that all overloads which + # take more than one argument are not applicable here. + if setter: + value_types =[] + if setter.signature and len(setter.signature._parameters) == 1: + value_types.append(setter.signature[0].type_hint) + if setter.hasOverloads(): + for overload in setter.overloads: + if overload.signature and len(overload.signature._parameters) == 1: + value_types.append( overload.signature[0].type_hint) + if len(value_types) == 1: + value_type = value_types[0] + elif value_types: + # This does not flatten already existing Unions, but this is allowed + value_type = f'Union[{", ".join(value_types)}]' + # We choose some default values if no types were found above. This may lead to wrong info, if the underlying getter and setter types are wrong to begin with + if value_type or return_type: + if not value_type: + value_type = return_type + elif not return_type: + # We probably should not choose the value_type as default if it contains a Union, + # but this might still be better than using Any. + return_type = value_type if prop.setter and prop.getter: - if value_type: + if value_type and return_type: stream.write(f'{indent}@property\n') - stream.write(f'{indent}def {prop.name}(self) -> {value_type}: ...\n') + stream.write(f'{indent}def {prop.name}(self) -> {return_type}: ...\n') stream.write(f'{indent}@{prop.name}.setter\n') stream.write(f'{indent}def {prop.name}(self, value: {value_type}, /) -> None: ...\n') else: @@ -591,7 +617,7 @@ def _generateProperty(self, klass: extractors.ClassDef, prop: Union[extractors.P elif prop.getter: if value_type: stream.write(f'{indent}@property\n') - stream.write(f'{indent}def {prop.name}(self) -> {value_type}: ...\n') + stream.write(f'{indent}def {prop.name}(self) -> {return_type}: ...\n') else: stream.write(f'{indent}{prop.name} = property({prop.getter})\n') elif prop.setter: From 2b119bf8e23d739845333da2406e3587f9783ba1 Mon Sep 17 00:00:00 2001 From: Geo5 Date: Thu, 24 Jul 2025 04:45:31 +0200 Subject: [PATCH 3/6] Fixes Uint32 typo --- etgtools/tweaker_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etgtools/tweaker_tools.py b/etgtools/tweaker_tools.py index cdbe4963c..9ae8c2e0a 100644 --- a/etgtools/tweaker_tools.py +++ b/etgtools/tweaker_tools.py @@ -401,7 +401,7 @@ def cleanType(self, type_name: str, is_input: bool = False) -> str: 'time_t': 'int', 'size_t': 'int', 'Int32': 'int', - 'UInt32': 'int', + 'Uint32': 'int', 'long': long_type, 'unsignedlong': long_type, 'ulong': long_type, From 7c461cab2076612c24070d30f4cc55cbe8eb62cf Mon Sep 17 00:00:00 2001 From: Geo5 Date: Thu, 24 Jul 2025 19:19:18 +0200 Subject: [PATCH 4/6] Ignore ignored overloads --- etgtools/pi_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etgtools/pi_generator.py b/etgtools/pi_generator.py index d7483134e..c442d6a87 100644 --- a/etgtools/pi_generator.py +++ b/etgtools/pi_generator.py @@ -591,8 +591,8 @@ def _generateProperty(self, klass: extractors.ClassDef, prop: Union[extractors.P value_types.append(setter.signature[0].type_hint) if setter.hasOverloads(): for overload in setter.overloads: - if overload.signature and len(overload.signature._parameters) == 1: - value_types.append( overload.signature[0].type_hint) + if not overload.ignored and overload.signature and len(overload.signature._parameters) == 1: + value_types.append(overload.signature[0].type_hint) if len(value_types) == 1: value_type = value_types[0] elif value_types: From e7221ae8fafb01ca47beb435f266c3fef72dd0f4 Mon Sep 17 00:00:00 2001 From: Geo5 Date: Thu, 24 Jul 2025 19:32:56 +0200 Subject: [PATCH 5/6] Delete type auto conversion cache in build.py - Clean up path joining --- build.py | 8 ++++++++ etgtools/tweaker_tools.py | 19 +++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/build.py b/build.py index 32f04724e..cae794207 100755 --- a/build.py +++ b/build.py @@ -1149,6 +1149,11 @@ def cmd_etg(options, args): # 2. Actually create the sip files # This is needed because each etg file is run in its own python invocation, so # the data is lost between them. + + # First delete the old cache if found + cacheFile = opj(cfg.ROOT_DIR, 'sip', 'gen', '__auto_conversion_cache__.json') + if os.path.isfile(cacheFile): + os.remove(cacheFile) is_newer = {} for script in etgfiles: sipfile = etg2sip(script) @@ -1170,6 +1175,9 @@ def cmd_etg(options, args): for script in etgfiles: if is_newer[script]: runcmd('"%s" %s %s' % (PYTHON, script, flags)) + # Delete the cache + if os.path.isfile(cacheFile): + os.remove(cacheFile) def cmd_sphinx(options, args): diff --git a/etgtools/tweaker_tools.py b/etgtools/tweaker_tools.py index 9ae8c2e0a..0a80ff663 100644 --- a/etgtools/tweaker_tools.py +++ b/etgtools/tweaker_tools.py @@ -20,7 +20,7 @@ import sys, os import copy import textwrap -from typing import NamedTuple, Optional, Tuple, Union +from typing import Final, NamedTuple, Optional, Tuple, Union isWindows = sys.platform.startswith('win') @@ -246,21 +246,25 @@ def removeWxPrefix(name): return name -def load_auto_conversions(destFile=None): +# Filename for type auto conversion cache file +_AUTO_CONVERSION_CACHE_FILE: Final = '__auto_conversion_cache__.json' + + +def load_auto_conversions(destFile: str | None = None) -> dict[str, Tuple[str, ...]]: """Load FixWxPrefix auto conversions from cache file if it exists.""" import json if not destFile: from buildtools.config import Config cfg = Config(noWxConfig=True) - phoenixRoot = cfg.ROOT_DIR - destFile = os.path.join(phoenixRoot, 'sip/gen', '__auto_conversion_cache__.json') + destFile = os.path.join(cfg.ROOT_DIR, 'sip', 'gen', _AUTO_CONVERSION_CACHE_FILE) # Need to merge existing data with current data if os.path.isfile(destFile): with open(destFile, 'r', encoding='utf-8') as f: return json.load(f) return {} + class FixWxPrefix(object): """ A mixin class that can help with removing the wx prefix, or changing it @@ -271,16 +275,15 @@ class FixWxPrefix(object): _auto_conversions: dict[str, Tuple[str, ...]] = load_auto_conversions() @classmethod - def cache_auto_conversions(cls, destFile=None): - """Save current auto conversions to a cache.""" + def cache_auto_conversions(cls, destFile: str | None = None) -> None: + """Save current auto conversions to a cache file.""" import json if not destFile: from buildtools.config import Config cfg = Config(noWxConfig=True) - phoenixRoot = cfg.ROOT_DIR - destFile = os.path.join(phoenixRoot, 'sip/gen', '__auto_conversion_cache__.json') + destFile = os.path.join(cfg.ROOT_DIR, 'sip', 'gen', _AUTO_CONVERSION_CACHE_FILE) # We overwrite the cache file, as we should have loaded the existing values when # initializing the class. with textfile_open(destFile, 'wt') as f: From 4d580ba58ec7d06a4a687d75fa0138c1d60b81e5 Mon Sep 17 00:00:00 2001 From: Geo5 Date: Tue, 29 Jul 2025 23:15:16 +0200 Subject: [PATCH 6/6] Fix for typing.Union 3.10 syntax --- etgtools/tweaker_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etgtools/tweaker_tools.py b/etgtools/tweaker_tools.py index 0a80ff663..633c82e63 100644 --- a/etgtools/tweaker_tools.py +++ b/etgtools/tweaker_tools.py @@ -250,7 +250,7 @@ def removeWxPrefix(name): _AUTO_CONVERSION_CACHE_FILE: Final = '__auto_conversion_cache__.json' -def load_auto_conversions(destFile: str | None = None) -> dict[str, Tuple[str, ...]]: +def load_auto_conversions(destFile: Optional[str] = None) -> dict[str, Tuple[str, ...]]: """Load FixWxPrefix auto conversions from cache file if it exists.""" import json if not destFile: @@ -275,7 +275,7 @@ class FixWxPrefix(object): _auto_conversions: dict[str, Tuple[str, ...]] = load_auto_conversions() @classmethod - def cache_auto_conversions(cls, destFile: str | None = None) -> None: + def cache_auto_conversions(cls, destFile: Optional[str] = None) -> None: """Save current auto conversions to a cache file.""" import json