Merge branch 'develop' into develop

Author: DavidWhittingham

arcpyext/TopicCategory.py

@@ -0,0 +1,29 @@
+# coding=utf-8
+"""
+This module contains an enumerated type representing ISO Topic Categories.
+"""
+from aenum import StrEnum
+
+
+class TopicCategory(StrEnum):
+    _init_ = "value standard_name display_name portal_name"
+
+    BIOTA = "002", "biota", "Biota", "/Categories/Biota"
+    BOUNDARIES = "003", "boundaries", "Boundaries", "/Categories/Boundaries"
+    CLIMATOLOGY_METEOROLOGY_ATMOSPHERE = "004", "climatologyMeteorologyAtmosphere", "Climatology, meteorology, and atmosphere", "/Categories/Climatology, meteorology, and atmosphere"
+    ECONOMY = "005", "economy", "Economy", "/Categories/Economy"
+    ELEVATION = "006", "elevation", "Elevation", "/Categories/Elevation"
+    ENVIRONMENT = "007", "environment", "Environment", "/Categories/Environment"
+    FARMING = "001", "farming", "Farming", "/Categories/Farming"
+    GEOSCIENTIFIC_INFORMATION = "008", "geoscientificInformation", "Geoscientific information", "/Categories/Geoscientific information"
+    HEALTH = "009", "health", "Health", "/Categories/Health"
+    IMAGERY_BASE_MAPS_EARTH_COVER = "010", "imageryBaseMapsEarthCover", "Imagery, basemaps, and Earth cover", "/Catagories/Imagery, basemaps, and Earth cover"
+    INLAND_WATERS = "012", "inlandWaters", "Inland waters", "/Categories/Inland waters"
+    INTELLIGENCE_MILITARY = "011", "intelligenceMilitary", "Intelligence and military", "/Categories/Intelligence and military"
+    LOCATION = "013", "location", "Location", "/Categories/Location"
+    OCEANS = "014", "oceans", "Oceans", "/Categories/Oceans"
+    PLANNING_CADASTRE = "015", "planningCadastre", "Planning and cadastre", "/Categories/Planning and cadastre"
+    SOCIETY = "016", "society", "Society", "/Categories/Society"
+    STRUCTURE = "017", "structure", "Structure", "/Categories/Structure"
+    TRANSPORTATION = "018", "transportation", "Transportation", "/Categories/Transportation"
+    UTILITIES_COMMUNICATION = "019", "utilitiesCommunication", "Utilities and communication", "/Categories/Utilities and communication"

arcpyext/__init__.py

@@ -19,4 +19,5 @@
 from . import toolbox
 from . import schematransform
 from . import mapping
-from . import publishing
+from . import publishing
+from .TopicCategory import TopicCategory

arcpyext/_json/json_enum.py

@@ -12,10 +12,10 @@
 # pylint: enable=wildcard-import,unused-wildcard-import,wrong-import-order,wrong-import-position
 
 # Standard library imports
-from enum import Enum # this is a backport package on Py2
+from aenum import Enum  # this is a backport package on Py2
 
-class JsonEnum(Enum):
 
+class JsonEnum(Enum):
     def __iter__(self):
         return iteritems(self._to_jsonable())
 

arcpyext/_patches/metadata/Metadata.py

@@ -0,0 +1,108 @@
+# coding=utf-8
+"""
+This module contains patches to Esri's *arcpy.metadata.Metadata* class.  These patches may insert functionality or fix 
+issues directly in the *arcpy.metadata.Metadata* class.
+"""
+import collections.abc
+import re
+
+import arcpy
+
+from lxml import etree as ET
+
+from ...TopicCategory import TopicCategory
+
+TOPIC_CATEGORIES_DISPLAY_NAME_TO_ENUM = {e.display_name.lower(): e for e in TopicCategory}
+TOPIC_CATEGORIES_STANDARD_NAME_TO_ENUM = {e.standard_name.lower(): e for e in TopicCategory}
+TOPIC_CATEGORIES_PORTAL_NAME_TO_ENUM = {e.portal_name.lower(): e for e in TopicCategory}
+
+
+def _str_to_topic_category(value):
+    try:
+        return TopicCategory(value)
+    except ValueError:
+        # not a valid input for the enum, try parsing text values
+        if not isinstance(value, str):
+            raise
+
+        value = value.lower()
+        for str_dict in [
+            TOPIC_CATEGORIES_DISPLAY_NAME_TO_ENUM, TOPIC_CATEGORIES_STANDARD_NAME_TO_ENUM,
+            TOPIC_CATEGORIES_PORTAL_NAME_TO_ENUM
+        ]:
+            enum_val = str_dict.get(value)
+            if enum_val:
+                return enum_val
+
+        # didn't find enum value based on string input either, re-raise ValueError
+        raise
+
+
+def add_topicCategories():
+    def topicCategories_getter(self):
+        metadata_xml = self.xml
+
+        if not metadata_xml:
+            # no XML, not supported, raise error
+            raise NotImplementedError("Metadata object currently has none or empty XML.")
+
+        metadata_xml_tree = ET.ElementTree(ET.fromstring(metadata_xml))
+        topic_categories_xpath = "/metadata[@xml:lang=\"en\"]/dataIdInfo/tpCat/TopicCatCd"
+
+        # get a list of topic elements
+        topic_category_elements = metadata_xml_tree.xpath(topic_categories_xpath)
+
+        if not topic_category_elements:
+            # no elmeents found, return empty list
+            return []
+
+        # convert each element value to a str
+        return [TopicCategory(value) for value in [e.attrib["value"] for e in topic_category_elements]]
+
+    def topicCategories_setter(self, value):
+        metadata_xml = self.xml
+
+        if not metadata_xml:
+            # no XML, not supported, raise error
+            raise NotImplementedError("Metadata object currently has none or empty XML.")
+
+        # process value into elements
+        if isinstance(value, str):
+            # presume we were supplied a one or more topic categorie as a string, delimited
+            value = [s.strip() for s in re.split(",|;", value)]
+        elif isinstance(value, TopicCategory):
+            # a single enumerated type has been given, put in a list
+            value = [value]
+
+        if value is None:
+            # value is none, make an empty list
+            value = []
+        elif not isinstance(value, collections.abc.Sequence):
+            raise ValueError("Input 'topicCategories' value is not valid.")
+
+        # value is now a sequence presumeably of either an enumerated type already, or a string representing the
+        # display name, the ISO name, or portal name.
+        # attempt to resolve each sequence value to an enum and then create element
+        value = [ET.Element("TopicCatCd", value=_str_to_topic_category(v).value) for v in value]
+
+        # set elements in XML
+        topic_categories_parent_xpath = "/metadata[@xml:lang=\"en\"]/dataIdInfo/tpCat"
+        metadata_xml_tree = ET.ElementTree(ET.fromstring(metadata_xml))
+        topic_category_parent_element = next(iter(metadata_xml_tree.xpath(topic_categories_parent_xpath)), None)
+        topic_category_parent_element.clear()
+        topic_category_parent_element.extend(value)
+
+        # save XML back to arcpy
+        self.xml = ET.tostring(metadata_xml_tree.getroot(), encoding="utf-8", pretty_print=False)
+
+    topicCategories_prop = property(topicCategories_getter, topicCategories_setter)
+
+    if hasattr(arcpy.metadata.Metadata, "topicCategories"):
+        # Esri have added a topicCategories property, bail out
+        return
+
+    # patch arcpy
+    arcpy.metadata.Metadata.topicCategories = topicCategories_prop
+
+
+add_topicCategories()

arcpyext/_patches/metadata/__init__.py

@@ -0,0 +1 @@
+from . import Metadata

arcpyext/_patches/patches.py

@@ -29,6 +29,7 @@ def apply():
 
     if sys.version_info[0] == 3:
         from . import _mp
+        from . import metadata
 
 
 def fix_mapping_versions():

arcpyext/_str/sql.py

@@ -13,7 +13,6 @@
 from sqlparse import engine
 from sqlparse import filters
 from sqlparse import filters, tokens
-from sqlparse.compat import text_type
 from sqlparse.exceptions import SQLParseError
 
 __all__ = ["format_def_query"]
@@ -53,11 +52,13 @@ def __init__(self, case=None):
             self.convert = lambda v: v
         else:
             case = case.lower()
-            ignore_quotes_keywords = "_ignore_quotes"
-            if case.endswith(ignore_quotes_keywords):
-                self.ignore_quotes = True
-                case = case[:-len(ignore_quotes_keywords)]
-            self.convert = getattr(text_type, case)
+
+            if case.startswith("upper"):
+                self.convert = str.upper
+            elif case.startswith("lower"):
+                self.convert = str.lower
+            else:
+                raise ValueError("Could not determine case conversion option for the following value: {}".format(case))
 
     def process(self, stream):
         for ttype, value in stream:
@@ -95,8 +96,7 @@ def format_def_query(def_query, encoding=None, **options):
 def validate_options(options):
     identifier_case = options.get("identifier_case")
     if identifier_case not in [
-            None, "upper", "upper_ignore_quotes", "lower", "lower_ignore_quotes", "capitalize",
-            "capitalize_ignore_quotes"
+        None, "upper", "upper_ignore_quotes", "lower", "lower_ignore_quotes", "capitalize", "capitalize_ignore_quotes"
     ]:
         raise SQLParseError("Invalid value for identifier_case: {0!r}".format(identifier_case))
 

arcpyext/_version.py

@@ -1,3 +1,3 @@
-__version__ = "0.7.24"
+__version__ = "0.7.27"
 __author__ = "David Whittingham; David Payne; Adam Kerz; Peter Reyne; Daniel Baternik; Chris Blanchfield; Gary Bagnall"
-__copyright__ = "Copyright (C) 2013-2022 David Whittingham"
+__copyright__ = "Copyright (C) 2013-2023 David Whittingham"

arcpyext/conversion/ToCsv.py

@@ -10,17 +10,6 @@
 install_aliases()
 # pylint: enable=wildcard-import,unused-wildcard-import,wrong-import-order,wrong-import-position
 
-# Standard library imports
-import csv
-import sys
-
-from io import BytesIO
-
-#from io import open
-
-# Third-party imports
-import arcpy
-
 # Local imports
 from ._ToCsvBase import ToCsvBase
 
@@ -31,21 +20,21 @@ class ToCsv(ToCsvBase):
 
     def feature_class(self, input_fc, output_fc, use_field_alias_as_column_header=False):
 
-        return super().feature_class(input_fc,
-                                     output_fc,
-                                     use_field_alias_as_column_header=use_field_alias_as_column_header)
+        return super().feature_class(
+            input_fc, output_fc, use_field_alias_as_column_header=use_field_alias_as_column_header
+        )
 
     def table(self, input_table, output_table, use_field_alias_as_column_header=False):
-        return super().table(input_table,
-                             output_table,
-                             use_field_alias_as_column_header=use_field_alias_as_column_header)
+        return super().table(
+            input_table, output_table, use_field_alias_as_column_header=use_field_alias_as_column_header
+        )
 
     def relationship_class(self, input_rel, output_rel):
         return super().relationship_class(input_rel, output_rel)
 
     def workspace(self, input_workspace, output_path, use_field_alias_as_column_header=False):
-        return super().workspace(input_workspace,
-                                 output_path,
-                                 use_field_alias_as_column_header=use_field_alias_as_column_header)
+        return super().workspace(
+            input_workspace, output_path, use_field_alias_as_column_header=use_field_alias_as_column_header
+        )
 
     #endregion

arcpyext/conversion/ToOfficeOpenXmlWorkbook.py

@@ -40,19 +40,22 @@ def feature_class(self, input_fc, output_workbook, use_field_alias_as_column_hea
             raise ValueError("output_workbook already exists.")
 
         # get input feature class description for copy process
-        d = arcpy.Describe(input_fc)
+        input_fc_desc = arcpy.Describe(input_fc)
 
-        if not d.dataType == "FeatureClass":
+        if not input_fc_desc.dataType == "FeatureClass":
             raise ValueError("input_fc is not of type 'FeatureClass'.")
 
         output_workbook = xlsxwriter.Workbook(str(output_workbook))
 
-        sheet_name = self._feature_class_default_name(d, output_workbook)
+        sheet_name = self._feature_class_default_name(input_fc_desc, output_workbook)
 
-        self._feature_class(d, output_workbook, sheet_name, use_field_alias_as_column_header)
+        self._feature_class(input_fc_desc, output_workbook, sheet_name, use_field_alias_as_column_header)
 
         output_workbook.close()
 
+        del input_fc_desc
+        arcpy.management.ClearWorkspaceCache()
+
     def table(self, input_table, output_workbook, use_field_alias_as_column_header=False):
         if not arcpy.Exists(input_table):
             raise ValueError("input_table does not exist.")
@@ -63,55 +66,68 @@ def table(self, input_table, output_workbook, use_field_alias_as_column_header=F
             raise ValueError("output_table already exists.")
 
         # get input feature class description for copy process
-        d = arcpy.Describe(input_table)
+        input_table_desc = arcpy.Describe(input_table)
 
-        if not d.dataType == "Table":
+        if not input_table_desc.dataType == "Table":
             raise ValueError("input_table is not of type 'Table'.")
 
         output_workbook = xlsxwriter.Workbook(str(output_workbook))
 
-        sheet_name = self._table_default_name(d, output_workbook)
+        sheet_name = self._table_default_name(input_table_desc, output_workbook)
 
-        self._table(d, output_workbook, sheet_name, use_field_alias_as_column_header)
+        self._table(input_table_desc, output_workbook, sheet_name, use_field_alias_as_column_header)
 
         output_workbook.close()
 
+        del input_table_desc
+        arcpy.management.ClearWorkspaceCache()
+
     def relationship_class(self, input_rel, output_rel):
         return super().relationship_class(input_rel, output_rel)
 
     def workspace(self, input_workspace, output_path, use_field_alias_as_column_header=False):
         if not arcpy.Exists(input_workspace):
             raise ValueError("input_workspace does not exist.")
 
-        d = arcpy.Describe(input_workspace)
+        input_workspace_desc = arcpy.Describe(input_workspace)
 
-        if not d.dataType == "Workspace":
+        if not input_workspace_desc.dataType == "Workspace":
             raise ValueError("input_workspace is not of type 'Workspace'.")
 
         # get output_path as a Path object
         output_path = Path(output_path).resolve()
 
         output_workbook = self._create_output_workspace(
-            output_path, use_field_alias_as_column_header=use_field_alias_as_column_header)
+            output_path, use_field_alias_as_column_header=use_field_alias_as_column_header
+        )
 
-        for c in d.children:
+        for c in input_workspace_desc.children:
             if c.dataType == 'FeatureClass':
-                self._feature_class(c,
-                                    output_workbook,
-                                    self._feature_class_default_name(c, output_workbook),
-                                    use_field_alias_as_column_header=use_field_alias_as_column_header)
+                self._feature_class(
+                    c,
+                    output_workbook,
+                    self._feature_class_default_name(c, output_workbook),
+                    use_field_alias_as_column_header=use_field_alias_as_column_header
+                )
             elif c.dataType == 'Table':
-                self._table(c,
-                            output_workbook,
-                            self._table_default_name(c, output_workbook),
-                            use_field_alias_as_column_header=use_field_alias_as_column_header)
+                self._table(
+                    c,
+                    output_workbook,
+                    self._table_default_name(c, output_workbook),
+                    use_field_alias_as_column_header=use_field_alias_as_column_header
+                )
             elif c.dataType == 'RelationshipClass':
-                self._relationship_class(c,
-                                         self._relationship_class_default_name(c, output_path),
-                                         use_field_alias_as_column_header=use_field_alias_as_column_header)
+                self._relationship_class(
+                    c,
+                    self._relationship_class_default_name(c, output_path),
+                    use_field_alias_as_column_header=use_field_alias_as_column_header
+                )
 
         output_workbook.close()
 
+        del input_workspace_desc
+        arcpy.management.ClearWorkspaceCache()
+
     #endregion
 
     #region Private overrides
@@ -167,10 +183,12 @@ def _dataset_to_ooxml(self, dataset, workbook, sheet_name, use_field_alias_as_co
         # create table layout, if any data was written
         if row_no > 0:
             header_attr = "name" if use_field_alias_as_column_header == False else "aliasName"
-            worksheet.add_table(0, 0, row_no - 1,
-                                len(fields) - 1, {"columns": [{
-                                    "header": getattr(f, header_attr)
-                                } for f in fields]})
+            worksheet.add_table(
+                0, 0, row_no - 1,
+                len(fields) - 1, {"columns": [{
+                    "header": getattr(f, header_attr)
+                } for f in fields]}
+            )
 
     def _get_default_name(self, desc, output_workbook, **kwargs):
         # get the default name for OOXML, respecting the 31 character limit and avoiding collisions