[eudsl-python-extras] better handle callable generics (#231)

This PR demos passing actual functions as generics (I don't have a fancy
name for this...). There's no real new functionality, just better name
mangling for such generics. Also drive-by changes:

- enable testing on `3.10`, `3.11`, `3.12` for all of our python-esque
tests (and fix `StrEnum` on `3.10` for eudsl-python-extras)
- move to using `bytecode` from pypi instead of source.
- fix some other GHA stuff

Author: makslevental

.github/workflows/build_mlir_python_bindings_wheel.yml

@@ -206,7 +206,7 @@ jobs:
           "windows-2022"
         ]
         python-version: [
-          # "3.10", "3.11", "3.12",
+          "3.10", "3.11", "3.12",
           "3.13", "3.14", "3.14t"
         ]
         include: [
@@ -230,6 +230,21 @@ jobs:
           - runs-on: macos-13
             python-version: "3.14t"
 
+          - runs-on: macos-14
+            python-version: "3.10"
+
+          - runs-on: macos-14
+            python-version: "3.11"
+
+          - runs-on: macos-14
+            python-version: "3.12"
+
+          - runs-on: macos-14
+            python-version: "3.13"
+
+          - runs-on: macos-14
+            python-version: "3.14"
+
     runs-on: ${{ matrix.runs-on }}
 
     name: "Test mlir-python-bindings ${{ matrix.name }} ${{ matrix.python-version }}"

.github/workflows/build_test_release_eudsl.yml

@@ -277,8 +277,8 @@ jobs:
                   # "macos-13",
                   "macos-14", "windows-2022"]
         python-version: [
-          # "3.9", "3.10", "3.11",
-          "3.12", "3.13"
+          "3.10", "3.11", "3.12",
+          "3.13", "3.14", "3.14t"
         ]
         include: [
           {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
@@ -291,6 +291,21 @@ jobs:
           - runs-on: macos-13
             python-version: "3.13"
 
+          - runs-on: macos-14
+            python-version: "3.10"
+
+          - runs-on: macos-14
+            python-version: "3.11"
+
+          - runs-on: macos-14
+            python-version: "3.12"
+
+          - runs-on: macos-14
+            python-version: "3.13"
+
+          - runs-on: macos-14
+            python-version: "3.14"
+
     runs-on: ${{ matrix.runs-on }}
 
     name: "Test tblgen ${{ matrix.name }} ${{ matrix.python-version }}"
@@ -306,7 +321,7 @@ jobs:
           submodules: false
 
       - name: "Install Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6.0.0
         with:
           python-version: "${{ matrix.python-version }}"
 
@@ -348,8 +363,11 @@ jobs:
           "windows-2022"
         ]
         python-version: [
+          # we only build 3.12 and up
           # "3.10", "3.11",
-          "3.12", "3.13"
+          "3.12", "3.13", "3.14",
+          # stable abi doesn't support 3.14t?
+          # "3.14t"
         ]
         include: [
           {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
@@ -362,6 +380,18 @@ jobs:
           - runs-on: macos-13
             python-version: "3.13"
 
+          - runs-on: macos-14
+            python-version: "3.10"
+
+          - runs-on: macos-14
+            python-version: "3.11"
+
+          - runs-on: macos-14
+            python-version: "3.12"
+
+          - runs-on: macos-14
+            python-version: "3.13"
+
     runs-on: ${{ matrix.runs-on }}
 
     name: "Test llvmpy ${{ matrix.name }} ${{ matrix.python-version }}"
@@ -377,7 +407,7 @@ jobs:
           submodules: false
 
       - name: "Install Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6.0.0
         with:
           python-version: "${{ matrix.python-version }}"
 
@@ -445,7 +475,7 @@ jobs:
 
     if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch'
 
-    needs: [test-eudsl-llvmpy, test-eudsl-tblgen]
+    needs: [release-eudsl]
 
     permissions:
       contents: read

.github/workflows/build_test_release_eudsl_python_extras.yml

@@ -94,7 +94,7 @@ jobs:
           "windows-2022"
         ]
         python-version: [
-          # "3.10", "3.11", "3.12",
+          "3.10", "3.11", "3.12",
           "3.13", "3.14", "3.14t"
         ]
         include: [
@@ -118,6 +118,21 @@ jobs:
           - runs-on: macos-13
             python-version: "3.14t"
 
+          - runs-on: macos-14
+            python-version: "3.10"
+
+          - runs-on: macos-14
+            python-version: "3.11"
+
+          - runs-on: macos-14
+            python-version: "3.12"
+
+          - runs-on: macos-14
+            python-version: "3.13"
+
+          - runs-on: macos-14
+            python-version: "3.14"
+
     runs-on: ${{ matrix.runs-on }}
 
     name: "Test eudsl-python-extras ${{ matrix.name }} ${{ matrix.python-version }}"

.github/workflows/clean_releases.yml

@@ -21,7 +21,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6.0.0
         with:
           python-version: "3.10"
 

projects/eudsl-llvmpy/src/llvm/function.py

@@ -65,16 +65,17 @@ def __init__(self, body_builder, *, return_type=None, entry_bb_name="entry"):
 
     def _is_decl(self):
         # magic constant found from looking at the code for an empty fn
+        if sys.version_info.minor == 14:
+            return self.body_builder.__code__.co_code == b"\x80\x00R\x00#\x00"
         if sys.version_info.minor == 13:
             return self.body_builder.__code__.co_code == b"\x95\x00g\x00"
-        elif sys.version_info.minor == 12:
+        if sys.version_info.minor == 12:
             return self.body_builder.__code__.co_code == b"\x97\x00y\x00"
-        elif sys.version_info.minor == 11:
+        if sys.version_info.minor == 11:
             return self.body_builder.__code__.co_code == b"\x97\x00d\x00S\x00"
-        elif sys.version_info.minor in {8, 9, 10}:
+        if sys.version_info.minor in {8, 9, 10}:
             return self.body_builder.__code__.co_code == b"d\x00S\x00"
-        else:
-            raise NotImplementedError(f"{sys.version_info.minor} not supported.")
+        raise NotImplementedError(f"{sys.version_info.minor} not supported.")
 
     def __str__(self):
         return str(f"{self.__class__} {self.__dict__}")

projects/eudsl-python-extras/mlir/extras/ast/util.py

@@ -92,7 +92,7 @@ def replace_closure(code, new_closure: Dict):
     LOAD_DEREF = opmap["LOAD_DEREF"]
 
     # get the orig localplus that will be loaded from by the orig bytecode LOAD_DEREF arg_i
-    localsplus, localsplus_name_to_idx = get_localsplus_name_to_idx(code)
+    localsplus, _localsplus_name_to_idx = get_localsplus_name_to_idx(code)
 
     # closure vars go into co_freevars
     new_code = code.replace(co_freevars=tuple(new_closure.keys()))
@@ -130,18 +130,23 @@ def reducer_override(self, obj):
         return super().reducer_override(obj)
 
 
+def copy_object(obj):
+    # see https://github.com/cloudpipe/cloudpickle/blob/f111f7ab6d302e9b1e2a568d0e4c574895db6a6e/cloudpickle/cloudpickle.py#L813
+    # for how this trick is accomplished (dill and pickle both fail to pickle eg generic typevars)
+    with io.BytesIO() as file:
+        cp = MLIRTypePickler(file)
+        cp.dump(obj)
+        obj = cloudpickle.loads(file.getvalue())
+    return obj
+
+
 # Based on http://stackoverflow.com/a/6528148/190597 (Glenn Maynard);
 # potentially more complete approach https://stackoverflow.com/a/56901529/9045206
 def copy_func(f, new_closure: Dict = None):
     if new_closure is not None:
         code, closure = replace_closure(f.__code__, new_closure)
     else:
-        # see https://github.com/cloudpipe/cloudpickle/blob/f111f7ab6d302e9b1e2a568d0e4c574895db6a6e/cloudpickle/cloudpickle.py#L813
-        # for how this trick is accomplished (dill and pickle both fail to pickle eg generic typevars)
-        with io.BytesIO() as file:
-            cp = MLIRTypePickler(file)
-            cp.dump(f.__closure__)
-            closure = cloudpickle.loads(file.getvalue())
+        closure = copy_object(f.__closure__)
         code = f.__code__
 
     g = types.FunctionType(