Added initial support for ArcGIS Pro 3

Author: DavidWhittingham

arcpyext/_native/_dotnet.py

@@ -14,8 +14,32 @@
 
 # Standard lib imports
 import os
+import re
+import sys
+
+from decimal import Decimal
 
 # Third-party imports
+import arcpy
+
+# clr import, must work out whether we are .NET 6 and setup coreclr first
+if sys.version_info[0] >= 3:
+    arcpy_install_info = arcpy.GetInstallInfo()
+    major_version = Decimal(re.search(r"^(\d+\.\d+)", arcpy.GetInstallInfo()['Version'], re.IGNORECASE).group(1))
+    from clr_loader import get_coreclr
+    from pythonnet import set_runtime
+
+    if arcpy_install_info["ProductName"] == "Server" and major_version >= Decimal(11.0):
+        # running against server, on Python 3, on version 11 or greater
+        dotnet_config = os.path.join(arcpy_install_info["InstallDir"], r"bin\ArcSOC.runtimeconfig.json")
+        rt = get_coreclr(dotnet_config)
+        set_runtime(rt)
+    elif major_version >= Decimal(3.0):
+        # if we're not on server, and the product version is 3 or greater (assume pro)
+        dotnet_config = os.path.join(arcpy_install_info["InstallDir"], r"bin\ArcGISPro.runtimeconfig.json")
+        rt = get_coreclr(dotnet_config)
+        set_runtime(rt)
+
 import clr
 
 # .NET Imports
@@ -70,7 +94,6 @@ def release_com_object(self, obj):
 def singlethreadapartment(func=None):
     def decorator(func):
         """Factory for creating the STA decorator."""
-
         def sta_wrapper(*args, **kwargs):
             """The STA wrapper function."""
 
@@ -113,8 +136,10 @@ def find_gac_assembly_path(simple_name):
         sorted_version_dirs = sorted(os.listdir(simple_assembly_path))
         len_sorted_version_dirs = len(sorted_version_dirs)
         if len_sorted_version_dirs > 0:
-            latest_version_dir = os.path.join(simple_assembly_path,
-                                              sorted(os.listdir(simple_assembly_path))[len_sorted_version_dirs - 1])
+            latest_version_dir = os.path.join(
+                simple_assembly_path,
+                sorted(os.listdir(simple_assembly_path))[len_sorted_version_dirs - 1]
+            )
             dll_path = os.path.join(latest_version_dir, simple_name + ".dll")
             if os.path.isfile(dll_path):
                 return dll_path

arcpyext/_version.py

@@ -1,3 +1,3 @@
-__version__ = "0.7.21"
+__version__ = "0.7.22"
 __author__ = "David Whittingham; David Payne; Adam Kerz; Peter Reyne; Daniel Baternik; Chris Blanchfield; Gary Bagnall"
 __copyright__ = "Copyright (C) 2013-2022 David Whittingham"

arcpyext/mapping/_cim/factories.py

@@ -9,16 +9,34 @@
 install_aliases()
 # pylint: enable=wildcard-import,unused-wildcard-import,wrong-import-order,wrong-import-position,import-error,no-name-in-module
 
+import json
+
 from .layers import ProFeatureLayer, ProGroupLayer, ProRasterLayer
 
-def create_layer(proj_zip, layer_xml):
-    """Factory function for creating a layer object bassed on the input XML."""
-    # determine XML type
+
+def create_layer(proj_zip, layer_string):
+    """Factory function for creating a layer object bassed on the input string."""
+    # determine layer type, try to pass as JSON first
     layer_obj = None
-    if layer_xml.startswith("<CIMFeatureLayer"):
-        layer_obj = ProFeatureLayer(proj_zip, layer_xml)
-    elif layer_xml.startswith("<CIMGroupLayer"):
-        layer_obj = ProGroupLayer(proj_zip, layer_xml)
-    elif layer_xml.startswith("<CIMRasterLayer"):
-        layer_obj = ProRasterLayer(proj_zip, layer_xml)
+
+    if layer_string.startswith("{"):
+        # layer is a JSON string, decode to determine type
+
+        layer_json = json.loads(layer_string)
+        layer_type = layer_json["type"]
+        if layer_type == "CIMFeatureLayer":
+            layer_obj = ProFeatureLayer(proj_zip, layer_string)
+        elif layer_type == "CIMGroupLayer":
+            layer_obj = ProGroupLayer(proj_zip, layer_string)
+        elif layer_type == "CIMRasterLayer":
+            layer_obj = ProRasterLayer(proj_zip, layer_string)
+    else:
+        # layer is probably XML, fallback to that
+        if layer_string.startswith("<CIMFeatureLayer"):
+            layer_obj = ProFeatureLayer(proj_zip, layer_string)
+        elif layer_string.startswith("<CIMGroupLayer"):
+            layer_obj = ProGroupLayer(proj_zip, layer_string)
+        elif layer_string.startswith("<CIMRasterLayer"):
+            layer_obj = ProRasterLayer(proj_zip, layer_string)
+
     return layer_obj

arcpyext/mapping/_cim/helpers.py

@@ -10,22 +10,25 @@
 # pylint: enable=wildcard-import,unused-wildcard-import,wrong-import-order,wrong-import-position,import-error,no-name-in-module
 
 
-def get_xml(zip_file, file_path):
+def read_file_in_zip(zip_file, file_path, decode="utf-8"):
     """Reads in an XML file as UTF-8 from a zip file."""
     with zip_file.open(file_path) as fp:
-        return fp.read().decode("utf-8")
+        if decode:
+            return fp.read().decode(decode)
+        else:
+            return fp.read()
 
 
 def passthrough_prop(prop_name, doc=None, obj_name="_cim_obj"):
     """Factory function for creating a property that passes through to the underlying ArcGIS Pro SDK object."""
-
     def _get(self):
         try:
             obj = getattr(self, obj_name)
             return getattr(obj, prop_name)
         except AttributeError as ae:
-            raise AttributeError("Unable to get the {} property on this instance of {}.".format(
-                prop_name, self.__class__.__name__))
+            raise AttributeError(
+                "Unable to get the {} property on this instance of {}.".format(prop_name, self.__class__.__name__)
+            )
 
     def _set(self, val):
         raise NotImplementedError("Property {} cannot be set".format(prop_name))

arcpyext/mapping/_cim/layers.py

@@ -83,18 +83,30 @@ def feature_table(self):
 
 
 class ProFeatureLayer(ProBasicFeatureLayer):
-    def __init__(self, proj_zip, xml_string):
-        super().__init__(proj_zip, CIMFeatureLayer.FromXml(xml_string))
+    def __init__(self, proj_zip, layer_string):
+        try:
+            super().__init__(proj_zip, CIMFeatureLayer.FromXml(layer_string))
+        except AttributeError:
+            # probably JSON, attempt that
+            super().__init__(proj_zip, CIMFeatureLayer.FromJson(layer_string))
 
 
 class ProRasterLayer(ProLayerBase):
-    def __init__(self, proj_zip, xml_string):
-        super().__init__(proj_zip, CIMRasterLayer.FromXml(xml_string))
+    def __init__(self, proj_zip, layer_string):
+        try:
+            super().__init__(proj_zip, CIMRasterLayer.FromXml(layer_string))
+        except AttributeError:
+            # probably JSON, attempt that
+            super().__init__(proj_zip, CIMRasterLayer.FromJson(layer_string))
 
 
 class ProGroupLayer(ProLayerBase):
-    def __init__(self, proj_zip, xml_string):
-        super().__init__(proj_zip, CIMGroupLayer.FromXml(xml_string))
+    def __init__(self, proj_zip, layer_string):
+        try:
+            super().__init__(proj_zip, CIMGroupLayer.FromXml(layer_string))
+        except AttributeError:
+            # probably JSON, attempt that
+            super().__init__(proj_zip, CIMGroupLayer.FromJson(layer_string))
 
     def _get_child_paths(self):
         return [cp[8:] for cp in self._cim_obj.Layers]

arcpyext/mapping/_cim/pro_map.py

@@ -13,7 +13,7 @@
 import arcpy
 
 # Local imports
-from .helpers import get_xml, passthrough_prop
+from .helpers import read_file_in_zip, passthrough_prop
 from .factories import create_layer
 from .tables import ProStandaloneTable
 
@@ -27,9 +27,13 @@ class ProMap(object):
     _spatial_reference = None
     _tables = None
 
-    def __init__(self, proj_zip, xml_string):
+    def __init__(self, proj_zip, map_string):
         self._proj_zip = proj_zip
-        self._cim_obj = CIMMap.FromXml(xml_string)
+        try:
+            self._cim_obj = CIMMap.FromXml(map_string)
+        except AttributeError:
+            # probably in JSON format, try that
+            self._cim_obj = CIMMap.FromJson(map_string)
 
     #region PROPERTIES
 
@@ -40,7 +44,7 @@ def __init__(self, proj_zip, xml_string):
     def spatial_reference(self):
         if not self._spatial_reference:
             self._spatial_reference = arcpy.SpatialReference(self._cim_obj.SpatialReference.Wkid)
-        
+
         return self._spatial_reference
 
     @property
@@ -57,32 +61,31 @@ def layers(self):
 
         # return a shallow copy so our internal list isn't altered
         return self._layers.copy()
-    
+
     @property
     def tables(self):
         if self._tables is None:
             self._tables = []
 
             # table paths are pre-pended with 'CIMPATH=', strip that to get the actual zip file path
-            table_paths = [tp[8:] for tp in self._cim_obj.StandaloneTables]
+            table_paths = [tp[8:] for tp in (self._cim_obj.StandaloneTables or [])]
 
             # build tables
             for tp in table_paths:
                 # get xml, determine type, create layer object, add to list
-                table_xml = get_xml(self._proj_zip, tp)
+                table_string = read_file_in_zip(self._proj_zip, tp)
 
-                self._tables.append(ProStandaloneTable(self._proj_zip, table_xml))
-        
-        return self._tables.copy()
+                self._tables.append(ProStandaloneTable(self._proj_zip, table_string))
 
+        return self._tables.copy()
 
     #endregion
 
     def _create_layers(self, layer_path):
         # get xml, determine type, create layer object, add to list
-        layer_xml = get_xml(self._proj_zip, layer_path)
+        layer_string = read_file_in_zip(self._proj_zip, layer_path)
 
-        layer_obj = create_layer(self._proj_zip, layer_xml)
+        layer_obj = create_layer(self._proj_zip, layer_string)
         self._layers.append(layer_obj)
 
         if layer_obj:

arcpyext/mapping/_cim/pro_project.py

@@ -19,7 +19,7 @@
 from ArcGIS.Core.CIM import CIMGISProject
 
 # Local imports
-from .helpers import get_xml
+from .helpers import read_file_in_zip
 from .pro_map import ProMap
 
 
@@ -53,17 +53,23 @@ def maps(self):
             map_paths = [pi.CatalogPath[8:] for pi in cimproject_items]
 
             # get XML for each map in archive, create ProMap object
-            self._pro_maps = [ProMap(self._proj_zip, get_xml(self._proj_zip, map_path)) for map_path in map_paths]
+            self._pro_maps = [
+                ProMap(self._proj_zip, read_file_in_zip(self._proj_zip, map_path)) for map_path in map_paths
+            ]
 
         # return a shallow copy so our internal list isn't altered
         return self._pro_maps.copy()
 
     @property
     def _cimgisproject(self):
-        if not "GISProject.xml" in self._cims:
-            self._cims["GISProject.xml"] = CIMGISProject.FromXml(get_xml(self._proj_zip, "GISProject.xml"))
-
-        return self._cims["GISProject.xml"]
+        if not "GISProject" in self._cims:
+            try:
+                self._cims["GISProject"] = CIMGISProject.FromXml(read_file_in_zip(self._proj_zip, "GISProject.xml"))
+            except AttributeError:
+                # probably running on ArcGIS Pro 3/ArcGIS Server 11, where the CIM on-disk format has moved to JSON, do that instead
+                self._cims["GISProject"] = CIMGISProject.FromJson(read_file_in_zip(self._proj_zip, "GISProject.json"))
+
+        return self._cims["GISProject"]
 
     #endregion
 

arcpyext/mapping/_mapping.py

@@ -34,13 +34,13 @@
 # Python-version dependent imports
 ARCPY_2 = sys.version_info[0] < 3
 if ARCPY_2:
-    # import to get access to 'private' helper methds
+    # import to get access to 'private' helper methods
     from . import _mapping2 as _mh
 
     # import all 'public' methods into current namespace
     from ._mapping2 import *
 else:
-    # import to get access to 'private' helper methds
+    # import to get access to 'private' helper methods
     from . import _mapping3 as _mh
 
     # import all 'public' methods into current namespace
@@ -121,8 +121,9 @@ def change_data_sources(mxd_or_proj, data_sources):
 
     try:
         if not len(errors) == 0:
-            raise ChangeDataSourcesError("A number of errors were encountered whilst change layer data sources.",
-                                         errors)
+            raise ChangeDataSourcesError(
+                "A number of errors were encountered whilst change layer data sources.", errors
+            )
 
         if document_was_opened:
             # If the document was opened by this function, the map has to be saved for changes to be persisted
@@ -183,8 +184,9 @@ def freeze(d):
     def match_new_data_source(layer_or_table):
         logger = _get_logger()
 
-        if layer_or_table == None or (layer_or_table.get("isGroupLayer") == True
-                                      and layer_or_table.get("isNetworkAnalystLayer") != True):
+        if layer_or_table == None or (
+            layer_or_table.get("isGroupLayer") == True and layer_or_table.get("isNetworkAnalystLayer") != True
+        ):
             # Layers that can't be described or are group layers (except NA layers) can't have their data updated
             return None
 
@@ -208,12 +210,14 @@ def match_new_data_source(layer_or_table):
 
                         if tokens["dataSet"] is not None and tokens["schema"] is not None:
                             logger.debug(1.11)
-                            new_conn["workspacePath"] = "{}\\{}.{}.gdb".format(new_conn["workspacePath"],
-                                                                               tokens["schema"], tokens["dataSet"])
+                            new_conn["workspacePath"] = "{}\\{}.{}.gdb".format(
+                                new_conn["workspacePath"], tokens["schema"], tokens["dataSet"]
+                            )
                         elif tokens["dataSet"] is not None:
                             logger.debug(1.12)
-                            new_conn["workspacePath"] = "{}\\{}.gdb".format(new_conn["workspacePath"],
-                                                                            tokens["dataSet"])
+                            new_conn["workspacePath"] = "{}\\{}.gdb".format(
+                                new_conn["workspacePath"], tokens["dataSet"]
+                            )
                         else:
                             logger.debug(1.13)
                             new_conn["workspacePath"] = "{}\\{}.gdb".format(new_conn["workspacePath"], tokens["table"])
@@ -224,10 +228,12 @@ def match_new_data_source(layer_or_table):
 
         return new_conn
 
-    return [{
-        "layers": [match_new_data_source(layer) for layer in df["layers"]],
-        "tables": [match_new_data_source(table) for table in df["tables"]]
-    } for df in map_desc["maps"]]
+    return [
+        {
+            "layers": [match_new_data_source(layer) for layer in df["layers"]],
+            "tables": [match_new_data_source(table) for table in df["tables"]]
+        } for df in map_desc["maps"]
+    ]
 
 
 def describe(mxd_or_proj):
@@ -315,14 +321,12 @@ def _match_layers(was_layers, now_layers):
         },
         {
             # same name and id, datasource changed
-            'fn':
-            lambda a, b: b
+            'fn': lambda a, b: b
             if same_id(a, b) and same_name(a, b) and not is_resolved_was(a) and not is_resolved_now(b) else None
         },
         {
             # same id and datasource, name changed
-            'fn':
-            lambda a, b: b
+            'fn': lambda a, b: b
             if same_id(a, b) and same_datasource(a, b) and not is_resolved_was(a) and not is_resolved_now(b) else None
         },
         {
@@ -332,8 +336,7 @@ def _match_layers(was_layers, now_layers):
         },
         {
             # same name and datasource, id changed
-            'fn':
-            lambda a, b: b
+            'fn': lambda a, b: b
             if same_name(a, b) and same_datasource(a, b) and not is_resolved_was(a) and not is_resolved_now(b) else None
         },
         {

requirements.test.txt

@@ -1,4 +1,4 @@
-numpy>=1.16,<1.17 ; python_version>='3' # OOTB numpy is broken inside virtual envs for some reason, force install a version for Py3
+# numpy>=1.16,<1.17 ; python_version>='3' # OOTB numpy is broken inside virtual envs for some reason, force install a version for Py3
 pylint==1.9.4 ; python_version<'3'
 pylint>=2 ; python_version>='3'
 pytest==4.6.11 ; python_version<'3' # last version that works on Py2