From c14409ffe11b9b74b7bc3edb460f18966c0709b9 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Wed, 17 Feb 2021 16:56:45 +0000 Subject: [PATCH 01/22] ConnectivityManager first pass. --- lib/iris/common/__init__.py | 78 ++ lib/iris/cube.py | 81 +- lib/iris/exceptions.py | 6 + lib/iris/experimental/ugrid.py | 1428 ++++++++++++-------------------- 4 files changed, 603 insertions(+), 990 deletions(-) diff --git a/lib/iris/common/__init__.py b/lib/iris/common/__init__.py index d8e8ba80ef..097c969904 100644 --- a/lib/iris/common/__init__.py +++ b/lib/iris/common/__init__.py @@ -8,7 +8,85 @@ """ +from typing import Mapping + from .lenient import * from .metadata import * from .mixin import * from .resolve import * + + +def filter_cf( + instances, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, +): + name = None + instance = None + + if isinstance(item, str): + name = item + else: + instance = item + + result = instances + + if name is not None: + result = [ + instance_ for instance_ in result if instance_.name() == name + ] + + if standard_name is not None: + result = [ + instance_ + for instance_ in result + if instance_.standard_name == standard_name + ] + + if long_name is not None: + result = [ + instance_ + for instance_ in result + if instance_.long_name == long_name + ] + + if var_name is not None: + result = [ + instance_ for instance_ in result if instance_.var_name == var_name + ] + + if attributes is not None: + if not isinstance(attributes, Mapping): + msg = ( + "The attributes keyword was expecting a dictionary " + "type, but got a %s instead." % type(attributes) + ) + raise ValueError(msg) + + def attr_filter(instance_): + return all( + k in instance_.attributes and instance_.attributes[k] == v + for k, v in attributes.items() + ) + + result = [instance_ for instance_ in result if attr_filter(instance_)] + + if instance is not None: + if hasattr(instance, "__class__") and instance.__class__ in ( + CoordMetadata, + DimCoordMetadata, + ): + target_metadata = instance + else: + target_metadata = instance.metadata + + result = [ + instance_ + for instance_ in result + if instance_.metadata == target_metadata + ] + + return result diff --git a/lib/iris/cube.py b/lib/iris/cube.py index a15951900b..893d5a94fe 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -13,7 +13,6 @@ from collections.abc import ( Iterable, Container, - Mapping, MutableMapping, Iterator, ) @@ -40,10 +39,9 @@ import iris.aux_factory from iris.common import ( CFVariableMixin, - CoordMetadata, CubeMetadata, - DimCoordMetadata, metadata_manager_factory, + filter_cf, ) import iris.coord_systems import iris.coords @@ -1639,14 +1637,6 @@ def coords( See also :meth:`Cube.coord()`. """ - name = None - coord = None - - if isinstance(name_or_coord, str): - name = name_or_coord - else: - coord = name_or_coord - coords_and_factories = [] if dim_coords in [True, None]: @@ -1656,33 +1646,14 @@ def coords( coords_and_factories += list(self.aux_coords) coords_and_factories += list(self.aux_factories) - if name is not None: - coords_and_factories = [ - coord_ - for coord_ in coords_and_factories - if coord_.name() == name - ] - - if standard_name is not None: - coords_and_factories = [ - coord_ - for coord_ in coords_and_factories - if coord_.standard_name == standard_name - ] - - if long_name is not None: - coords_and_factories = [ - coord_ - for coord_ in coords_and_factories - if coord_.long_name == long_name - ] - - if var_name is not None: - coords_and_factories = [ - coord_ - for coord_ in coords_and_factories - if coord_.var_name == var_name - ] + coords_and_factories = filter_cf( + coords_and_factories, + item=name_or_coord, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + ) if axis is not None: axis = axis.upper() @@ -1693,26 +1664,6 @@ def coords( if guess_axis(coord_) == axis ] - if attributes is not None: - if not isinstance(attributes, Mapping): - msg = ( - "The attributes keyword was expecting a dictionary " - "type, but got a %s instead." % type(attributes) - ) - raise ValueError(msg) - - def attr_filter(coord_): - return all( - k in coord_.attributes and coord_.attributes[k] == v - for k, v in attributes.items() - ) - - coords_and_factories = [ - coord_ - for coord_ in coords_and_factories - if attr_filter(coord_) - ] - if coord_system is not None: coords_and_factories = [ coord_ @@ -1720,20 +1671,6 @@ def attr_filter(coord_): if coord_.coord_system == coord_system ] - if coord is not None: - if hasattr(coord, "__class__") and coord.__class__ in ( - CoordMetadata, - DimCoordMetadata, - ): - target_metadata = coord - else: - target_metadata = coord.metadata - coords_and_factories = [ - coord_ - for coord_ in coords_and_factories - if coord_.metadata == target_metadata - ] - if contains_dimension is not None: coords_and_factories = [ coord_ diff --git a/lib/iris/exceptions.py b/lib/iris/exceptions.py index 1c05d13163..12d24ef70f 100644 --- a/lib/iris/exceptions.py +++ b/lib/iris/exceptions.py @@ -39,6 +39,12 @@ class AncillaryVariableNotFoundError(KeyError): pass +class ConnectivityNotFoundError(KeyError): + """Raised when a search yields no connectivities.""" + + pass + + class CoordinateMultiDimError(ValueError): """Raised when a routine doesn't support multi-dimensional coordinates.""" diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 002c40952f..129b69cb17 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -10,18 +10,18 @@ """ -from collections import Mapping, namedtuple +from abc import ABC, abstractmethod +from collections import namedtuple from functools import wraps +from warnings import warn import dask.array as da import numpy as np from .. import _lazy_data as _lazy +from ..common import filter_cf from ..common.metadata import ( - _hexdigest, BaseMetadata, - CoordMetadata, - DimCoordMetadata, metadata_manager_factory, SERVICES, SERVICES_COMBINE, @@ -29,48 +29,16 @@ SERVICES_DIFFERENCE, ) from ..common.lenient import _lenient_service as lenient_service -from ..common.mixin import CFVariableMixin -from ..config import get_logger -from ..coords import _DimensionalMetadata, AuxCoord -from ..exceptions import CoordinateNotFoundError -from ..util import guess_coord_axis +from ..coords import _DimensionalMetadata +from .. import exceptions __all__ = [ "Connectivity", "ConnectivityMetadata", - "Mesh1DCoords", - "Mesh2DCoords", - "MeshEdgeCoords", - "MeshFaceCoords", - "MeshNodeCoords", - "MeshMetadata", ] -# Configure the logger. -logger = get_logger(__name__, fmt="[%(cls)s.%(funcName)s]") - - -# Mesh dimension names namedtuples. -Mesh1DNames = namedtuple("Mesh1DNames", ["node_dimension", "edge_dimension"]) -Mesh2DNames = namedtuple( - "Mesh2DNames", ["node_dimension", "edge_dimension", "face_dimension"] -) - -# Mesh coordinate manager namedtuples. -Mesh1DCoords = namedtuple( - "Mesh1DCoords", ["node_x", "node_y", "edge_x", "edge_y"] -) -Mesh2DCoords = namedtuple( - "Mesh2DCoords", - ["node_x", "node_y", "edge_x", "edge_y", "face_x", "face_y"], -) -MeshNodeCoords = namedtuple("MeshNodeCoords", ["node_x", "node_y"]) -MeshEdgeCoords = namedtuple("MeshEdgeCoords", ["edge_x", "edge_y"]) -MeshFaceCoords = namedtuple("MeshFaceCoords", ["face_x", "face_y"]) - - class Connectivity(_DimensionalMetadata): """ A CF-UGRID topology connectivity, describing the topological relationship @@ -773,932 +741,577 @@ def equal(self, other, lenient=None): return super().equal(other, lenient=lenient) -class Mesh(CFVariableMixin): - """ - - .. todo:: - - .. questions:: - - - decide on the verbose/succinct version of __str__ vs __repr__ - - .. notes:: +class _MeshConnectivityManagerMixin(ABC): + REQUIRED = () + OPTIONAL = () + NDIM = NotImplemented + MembersTuple = NotImplemented - - the mesh is location agnostic + @abstractmethod + def __init__(self, *connectivities): + cf_roles = [c.cf_role for c in connectivities] + for requisite in self.REQUIRED: + if requisite not in cf_roles: + message = f"{self.NDIM}D meshes require a {requisite}." + raise ValueError(message) - - no need to support volume at mesh level, yet + self.valid_roles = self.REQUIRED + self.OPTIONAL + self._members = {} + self.add(*connectivities) - - topology_dimension - - use for fast equality between Mesh instances - - checking connectivity dimensionality, specifically the highest dimensonality of the - "geometric element" being added i.e., reference the src_location/tgt_location - - used to honour and enforce the minimum UGRID connectivity contract - - - support pickling - - - copy is off the table!! - - - MeshCoord.guess_points() - - MeshCoord.to_AuxCoord() - - - don't provide public methods to return the coordinate and connectivity - managers - - - validate both managers contents e.g., shape? more...? - - """ - - # TBD: for volume and/or z-axis support include axis "z" and/or dimension "3" - AXES = ("x", "y") - TOPOLOGY_DIMENSIONS = (1, 2) - - def __init__( - self, - topology_dimension, - node_coords_and_axes, - standard_name=None, - long_name=None, - var_name=None, - units=None, - attributes=None, - edge_coords_and_axes=None, - face_coords_and_axes=None, - # connectivities=None, - node_dimension=None, - edge_dimension=None, - face_dimension=None, - ): - # TODO: support volumes. - # TODO: support (coord, "z") - - self._metadata_manager = metadata_manager_factory(MeshMetadata) - - # topology_dimension is read-only, so assign directly to the metadata manager - if topology_dimension not in self.TOPOLOGY_DIMENSIONS: - emsg = f"Expected 'topology_dimension' in range {self.TOPOLOGY_DIMENSIONS!r}, got {topology_dimension!r}." - raise ValueError(emsg) - self._metadata_manager.topology_dimension = topology_dimension - - # TBD: these are strings, if None is provided then assign the default string. - self.node_dimension = node_dimension - self.edge_dimension = edge_dimension - self.face_dimension = face_dimension - - # assign the metadata to the metadata manager - self.standard_name = standard_name - self.long_name = long_name - self.var_name = var_name - self.units = units - self.attributes = attributes - - # based on the topology_dimension, create the appropriate coordinate manager - def normalise(location, axis): - result = str(axis).lower() - if result not in self.AXES: - emsg = f"Invalid axis specified for {location} coordinate {coord.name()!r}, got {axis!r}." - raise ValueError(emsg) - return f"{location}_{axis}" - - kwargs = {} - for coord, axis in node_coords_and_axes: - kwargs[normalise("node", axis)] = coord - if edge_coords_and_axes is not None: - for coord, axis in edge_coords_and_axes: - kwargs[normalise("edge", axis)] = coord - if face_coords_and_axes is not None: - for coord, axis in face_coords_and_axes: - kwargs[normalise("face", axis)] = coord - - # check the UGRID minimum requirement for coordinates - if "node_x" not in kwargs: - emsg = ( - "Require a node coordinate that is x-axis like to be provided." - ) - raise ValueError(emsg) - if "node_y" not in kwargs: - emsg = ( - "Require a node coordinate that is y-axis like to be provided." - ) - raise ValueError(emsg) - - if self.topology_dimension == 1: - self._coord_manager = _Mesh1DCoordinateManager(**kwargs) - elif self.topology_dimension == 2: - self._coord_manager = _Mesh2DCoordinateManager(**kwargs) - else: - emsg = f"Unsupported 'topology_dimension', got {topology_dimension!r}." - raise NotImplementedError(emsg) - - # based on the topology_dimension, create the appropriate connectivity manager - # self._connectivity_manager = ... - - def __eq__(self, other): - # TBD - return NotImplemented + def __iter__(self): + for member, connectivity in self._members.items(): + yield member, connectivity def __getstate__(self): - # TBD pass - def __ne__(self, other): - # TBD - return NotImplemented - - def __repr__(self): - # TBD - args = [] - return f"{self.__class__.__name__}({', '.join(args)})" - def __setstate__(self, state): - # TBD pass - def __str__(self): - # TBD - args = [] - return f"{self.__class__.__name__}({', '.join(args)})" - - @property - def all_coords(self): - return self._coord_manager.all_members - - @property - def edge_dimension(self): - return self._edge_dimension - - @edge_dimension.setter - def edge_dimension(self, name): - if not name or not isinstance(name, str): - self._edge_dimension = f"Mesh{self.topology_dimension}d_edge" - else: - self._edge_dimension = name - @property - def edge_coords(self): - return self._coord_manager.edge_coords - - @property - def face_dimension(self): - return self._face_dimension - - @face_dimension.setter - def face_dimension(self, name): - if not name or not isinstance(name, str): - self._face_dimension = f"Mesh{self.topology_dimension}d_face" - else: - self._face_dimension = name - - @property - def face_coords(self): - return self._coord_manager.face_coords - - @property - def node_dimension(self): - return self._node_dimension - - @node_dimension.setter - def node_dimension(self, name): - if not name or not isinstance(name, str): - self._node_dimension = f"Mesh{self.topology_dimension}d_node" - else: - self._node_dimension = name - - @property - def node_coords(self): - return self._coord_manager.node_coords - - # @property - # def all_connectivities(self): - # # return a namedtuple - # # conns = mesh.all_connectivities - # # conns.edge_node, conns.boundary_node - # pass - # - # @property - # def face_node_connectivity(self): - # # required - # return self._connectivity_manager.face_node - # - # @property - # def edge_node_connectivity(self): - # # optionally required - # return self._connectivity_manager.edge_node - # - # @property - # def face_edge_connectivity(self): - # # optional - # return self._connectivity_manager.face_edge - # - # @property - # def face_face_connectivity(self): - # # optional - # return self._connectivity_manager.face_face - # - # @property - # def edge_face_connectivity(self): - # # optional - # return self._connectivity_manager.edge_face - # - # @property - # def boundary_node_connectivity(self): - # # optional - # return self._connectivity_manager.boundary_node - - def add_coords( - self, - node_x=None, - node_y=None, - edge_x=None, - edge_y=None, - face_x=None, - face_y=None, - ): - self._coord_manager.add( - node_x=node_x, - node_y=node_y, - edge_x=edge_x, - edge_y=edge_y, - face_x=face_x, - face_y=face_y, - ) + @abstractmethod + def all_members(self): + return NotImplemented - # def add_connectivities(self, *args): - # # this supports adding a new connectivity to the manager, but also replacing an existing connectivity - # self._connectivity_manager.add(*args) - - # def connectivities( - # self, - # name_or_coord=None, - # standard_name=None, - # long_name=None, - # var_name=None, - # attributes=None, - # node=False, - # edge=False, - # face=False, - # ): - # pass - - # def connectivity(self, ...): - # pass - - def coord( + def filter( self, item=None, standard_name=None, long_name=None, var_name=None, attributes=None, - axis=None, node=None, edge=None, face=None, ): - return self._coord_manager.filter( + # see Cube.coords for relevant patterns + result = filter_cf( + self._members.values(), item=item, standard_name=standard_name, long_name=long_name, var_name=var_name, attributes=attributes, - axis=axis, - node=node, - edge=edge, - face=face, ) - def coords( - self, - item=None, - standard_name=None, - long_name=None, - var_name=None, - attributes=None, - axis=None, - node=False, - edge=False, - face=False, - ): - return self._coord_manager.filters( - item=item, - standard_name=standard_name, - long_name=long_name, - var_name=var_name, - attributes=attributes, - axis=axis, - node=node, - edge=edge, - face=face, - ) + def location_filter(instances, parameter, location_name): + result = instances + if parameter is True: + result = [ + instance_ + for instance_ in result + if location_name + in (instance_.src_location, instance_.tgt_location) + ] + elif parameter is False: + result = [ + instance_ + for instance_ in result + if location_name + not in (instance_.src_location, instance_.tgt_location) + ] + + return result + + for parameter, location_name in ( + (node, "node"), + (edge, "edge"), + (face, "face"), + ): + result = location_filter(result, parameter, location_name) - # def remove_connectivities(self, ...): - # # needs to respect the minimum UGRID contract - # self._connectivity_manager.remove(...) + return result - def remove_coords( + def filter_single(self, **kwargs): + result = self.filter(**kwargs) + if len(result) > 1: + message = ( + f"Expected to find exactly 1 connectivity, but found " + f"{len(result)}. They were: {[c.name for c in result]}." + ) + raise exceptions.ConnectivityNotFoundError(message) + elif len(result) == 0: + item = kwargs["item"] + _name = item + if item is not None: + if not isinstance(item, str): + _name = item.name() + bad_name = ( + _name or kwargs["standard_name"] or kwargs["long_name"] or "" + ) + message = ( + f"Expected to find exactly 1 {bad_name} connectivity, " + f"but found none." + ) + raise exceptions.ConnectivityNotFoundError(message) + + return result[0] + + def add(self, *connectivities): + # Since Connectivity classes include their cf_role, no setters will be + # provided, just a means to add one or more connectivities to the + # manager. + # No warning is raised for duplicate cf_roles - user is trusted to + # validate their outputs. + add_dict = {} + for connectivity in connectivities: + assert isinstance(connectivity, Connectivity) + cf_role = connectivity.cf_role + if cf_role not in self.valid_roles: + message = ( + f"Connectivity not added. Got cf_role={cf_role} . " + f"Expected one of: {self.valid_roles} ." + ) + warn(message) + else: + add_dict[cf_role] = connectivity + + # Validate shapes. + proposed_members = {**self._members, **add_dict} + locations = set([c.src_location for c in proposed_members.values()]) + for location in locations: + counts = [ + len(c.indices_by_src()) + for c in proposed_members.values() + if c.src_location == location + ] + # Check is list values are identical. + if not counts.count(counts[0]) == len(counts): + message = ( + f"Invalid Connectivities provided - inconsistent " + f"{location} counts." + ) + raise ValueError(message) + + self._members = proposed_members + + def remove( self, item=None, standard_name=None, long_name=None, var_name=None, attributes=None, - axis=None, node=None, edge=None, face=None, ): - self._coord_manager.remove( + # use logging/warning to flag items not removed - highlight in doc-string + # don't raise an exception + removal_list = self.filter( item=item, standard_name=standard_name, long_name=long_name, var_name=var_name, attributes=attributes, - axis=axis, node=node, edge=edge, face=face, ) + removal_dict = {c.cf_role: c for c in removal_list} + for cf_role in self.REQUIRED: + if removal_dict.pop(cf_role, None): + message = ( + f"Connectivity not removed: {cf_role} - required " + f"for a valid {self.NDIM}D Mesh." + ) + warn(message) - def xml_element(self): - # TBD - pass - - # the MeshCoord will always have bounds, perhaps points. However the MeshCoord.guess_points() may - # be a very useful part of its behaviour. - # after using MeshCoord.guess_points(), the user may wish to add the associated MeshCoord.points into - # the Mesh as face_coordinates. - - # def to_AuxCoord(self, location, axis): - # # factory method - # # return the lazy AuxCoord(...) for the given location and axis - # - # def to_AuxCoords(self, location): - # # factory method - # # return the lazy AuxCoord(...), AuxCoord(...) - # - # def to_MeshCoord(self, location, axis): - # # factory method - # # return MeshCoord(..., location=location, axis=axis) - # # use Connectivity.indices_by_src() for fetching indices. - # - # def to_MeshCoords(self, location): - # # factory method - # # return MeshCoord(..., location=location, axis="x"), MeshCoord(..., location=location, axis="y") - # # use Connectivity.indices_by_src() for fetching indices. - - def dimension_names_reset(self, node=False, edge=False, face=False): - if node: - self.node_dimension = None - if edge: - self.edge_dimension = None - if face: - self.face_dimension = None - if self.topology_dimension == 1: - result = Mesh1DNames(self.node_dimension, self.edge_dimension) - else: - result = Mesh2DNames( - self.node_dimension, self.edge_dimension, self.face_dimension - ) - return result - - def dimension_names(self, node=None, edge=None, face=None): - if node: - self.node_dimension = node - if edge: - self.edge_dimension = edge - if face: - self.face_dimension = face - if self.topology_dimension == 1: - result = Mesh1DNames(self.node_dimension, self.edge_dimension) - else: - result = Mesh2DNames( - self.node_dimension, self.edge_dimension, self.node_dimension - ) - return result - - @property - def cf_role(self): - return "mesh_topology" - - @property - def topology_dimension(self): - return self._metadata_manager.topology_dimension - - -class _Mesh1DCoordinateManager: - """ - - TBD: require clarity on coord_systems validation - TBD: require clarity on __eq__ support - TBD: rationalise self.coords() logic with other manager and Cube - - """ - - REQUIRED = ( - "node_x", - "node_y", - ) - OPTIONAL = ( - "edge_x", - "edge_y", - ) + for cf_role in removal_dict.keys(): + del self._members[cf_role] - def __init__(self, node_x, node_y, edge_x=None, edge_y=None): - # initialise all the coordinates - self.ALL = self.REQUIRED + self.OPTIONAL - self._members = {member: None for member in self.ALL} + return removal_dict - # required coordinates - self.node_x = node_x - self.node_y = node_y - # optional coordinates - self.edge_x = edge_x - self.edge_y = edge_y + def __repr__(self): + class_name = type(self).__name__ + content = ", ".join( + f"{member}={connectivity}" for member, connectivity in self + ) + return ", ".join((class_name, content)) def __eq__(self, other): - # TBD - return NotImplemented - - def __getstate__(self): - # TBD - pass - - def __iter__(self): - for item in self._members.items(): - yield item + # Full equality could be MASSIVE, so we want to avoid that. + # Ideally we want a mesh signature from LFRic for comparison, although this would + # limit Iris' relevance outside MO. + # TL;DR: unknown quantity. + raise NotImplementedError def __ne__(self, other): - # TBD - return NotImplemented - - def __repr__(self): - args = [ - f"{member}={coord!r}" - for member, coord in self - if coord is not None - ] - return f"{self.__class__.__name__}({', '.join(args)})" - - def __setstate__(self, state): - # TBD - pass - - def __str__(self): - args = [ - f"{member}=True" for member, coord in self if coord is not None - ] - return f"{self.__class__.__name__}({', '.join(args)})" - - @staticmethod - def _filters( - members, - item=None, - standard_name=None, - long_name=None, - var_name=None, - attributes=None, - axis=None, - ): - """ - TDB: support coord_systems? - - """ - name = None - coord = None - - if isinstance(item, str): - name = item - else: - coord = item - - if name is not None: - members = {k: v for k, v in members.items() if v.name() == name} - - if standard_name is not None: - members = { - k: v - for k, v in members.items() - if v.standard_name == standard_name - } - - if long_name is not None: - members = { - k: v for k, v in members.items() if v.long_name == long_name - } - - if var_name is not None: - members = { - k: v for k, v in members.items() if v.var_name == var_name - } - - if axis is not None: - axis = axis.upper() - members = { - k: v for k, v in members.items() if guess_coord_axis(v) == axis - } - - if attributes is not None: - if not isinstance(attributes, Mapping): - emsg = ( - "The attributes keyword was expecting a dictionary " - f"type, but got a {type(attributes)} instead." - ) - raise ValueError(emsg) - - def _filter(coord): - return all( - k in coord.attributes - and _hexdigest(coord.attributes[k]) == _hexdigest(v) - for k, v in attributes.items() - ) - - members = {k: v for k, v in members.items() if _filter(v)} - - if coord is not None: - if hasattr(coord, "__class__") and coord.__class__ in ( - CoordMetadata, - DimCoordMetadata, - ): - target_metadata = coord - else: - target_metadata = coord.metadata - - members = { - k: v - for k, v in members.items() - if v.metadata == target_metadata - } - - return members - - def _remove(self, **kwargs): - result = {} - members = self.filters(**kwargs) - - for member in members.keys(): - if member in self.REQUIRED: - dmsg = f"Ignoring request to remove required coordinate {member!r}" - logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) - else: - result[member] = members[member] - setattr(self, member, None) - - return result - - def _setter(self, location, axis, coord, shape): - axis = axis.lower() - member = f"{location}_{axis}" - - # enforce the UGRID minimum coordinate requirement - if location == "node" and coord is None: - emsg = ( - f"{member!r} is a required coordinate, cannot set to 'None'." - ) - raise ValueError(emsg) - - if coord is not None: - if not isinstance(coord, AuxCoord): - emsg = f"{member!r} requires to be an 'AuxCoord', got {type(coord)}." - raise TypeError(emsg) - - guess_axis = guess_coord_axis(coord) - - if guess_axis and guess_axis.lower() != axis: - emsg = f"{member!r} requires a {axis}-axis like 'AuxCoord', got a {guess_axis.lower()}-axis like." - raise TypeError(emsg) - - if coord.climatological: - emsg = f"{member!r} cannot be a climatological 'AuxCoord'." - raise TypeError(emsg) - - if shape is not None and coord.shape != shape: - emsg = f"{member!r} requires to have shape {shape!r}, got {coord.shape!r}." - raise ValueError(emsg) + # See __eq__ + raise NotImplementedError - self._members[member] = coord - def _shape(self, location): - coord = getattr(self, f"{location}_x") - shape = coord.shape if coord is not None else None - if shape is None: - coord = getattr(self, f"{location}_y") - if coord is not None: - shape = coord.shape - return shape +# keep an eye on the __init__ inheritance +class _Mesh1DConnectivityManager(_MeshConnectivityManagerMixin): + REQUIRED = ("edge_node_connectivity",) + OPTIONAL = () + NDIM = 1 + MembersTuple = namedtuple("Mesh1DConnectivities", ["edge_node"]) - @property - def _edge_shape(self): - return self._shape(location="edge") - - @property - def _node_shape(self): - return self._shape(location="node") + def __init__(self, *connectivities): + super().__init__(*connectivities) + # TODO: debatable whether one couldn't just use self.filter(). @property def all_members(self): - return Mesh1DCoords(**self._members) - - @property - def edge_coords(self): - return MeshEdgeCoords(edge_x=self.edge_x, edge_y=self.edge_y) + return self.MembersTuple(self.edge_node) @property - def edge_x(self): - return self._members["edge_x"] + def edge_node(self): + return self._members["edge_node_connectivity"] - @edge_x.setter - def edge_x(self, coord): - self._setter( - location="edge", axis="x", coord=coord, shape=self._edge_shape - ) - @property - def edge_y(self): - return self._members["edge_y"] - - @edge_y.setter - def edge_y(self, coord): - self._setter( - location="edge", axis="y", coord=coord, shape=self._edge_shape - ) +class _Mesh2DConnectivityManager(_MeshConnectivityManagerMixin): + REQUIRED = ("face_node_connectivity",) + OPTIONAL = ( + "edge_node_connectivity", + "face_edge_connectivity", + "face_face_connectivity", + "edge_face_connectivity", + "boundary_node_connectivity", + ) + NDIM = 2 + MembersTuple = namedtuple( + "Mesh2DConnectivities", + [ + "face_node", + "edge_node", + "face_edge", + "face_face", + "edge_face", + "boundary_node", + ], + ) - @property - def node_coords(self): - return MeshNodeCoords(node_x=self.node_x, node_y=self.node_y) + def __init__(self, *connectivities): + super().__init__(*connectivities) + # TODO: debatable whether one couldn't just use self.filter(). @property - def node_x(self): - return self._members["node_x"] - - @node_x.setter - def node_x(self, coord): - self._setter( - location="node", axis="x", coord=coord, shape=self._node_shape + def all_members(self): + return self.MembersTuple( + self.face_node, + self.edge_node, + self.face_edge, + self.face_face, + self.edge_face, + self.boundary_node, ) @property - def node_y(self): - return self._members["node_y"] - - @node_y.setter - def node_y(self, coord): - self._setter( - location="node", axis="y", coord=coord, shape=self._node_shape - ) - - def _add(self, coords): - member_x, member_y = coords._fields - - # deal with the special case where both members are changing - if coords[0] is not None and coords[1] is not None: - cache_x = self._members[member_x] - cache_y = self._members[member_y] - self._members[member_x] = None - self._members[member_y] = None - - try: - setattr(self, member_x, coords[0]) - setattr(self, member_y, coords[1]) - except (TypeError, ValueError): - # restore previous valid state - self._members[member_x] = cache_x - self._members[member_y] = cache_y - # now, re-raise the exception - raise - else: - # deal with the case where one or no member is changing - if coords[0] is not None: - setattr(self, member_x, coords[0]) - if coords[1] is not None: - setattr(self, member_y, coords[1]) - - def add(self, node_x=None, node_y=None, edge_x=None, edge_y=None): - """ - use self.remove(edge_x=True) to remove a coordinate e.g., using the - pattern self.add(edge_x=None) will not remove the edge_x coordinate - - """ - self._add(MeshNodeCoords(node_x, node_y)) - self._add(MeshEdgeCoords(edge_x, edge_y)) - - def filter(self, **kwargs): - result = self.filters(**kwargs) - - if len(result) > 1: - names = ", ".join( - f"{member}={coord!r}" for member, coord in result.items() - ) - emsg = ( - f"Expected to find exactly 1 coordinate, but found {len(result)}. " - f"They were: {names}." - ) - raise CoordinateNotFoundError(emsg) - - if len(result) == 0: - item = kwargs["item"] - if item is not None: - if not isinstance(item, str): - item = item.name() - name = ( - item - or kwargs["standard_name"] - or kwargs["long_name"] - or kwargs["var_name"] - or None - ) - name = "" if name is None else f"{name!r} " - emsg = ( - f"Expected to find exactly 1 {name}coordinate, but found none." - ) - raise CoordinateNotFoundError(emsg) - - return result - - def filters( - self, - item=None, - standard_name=None, - long_name=None, - var_name=None, - attributes=None, - axis=None, - node=None, - edge=None, - face=None, - ): - # rationalise the tri-state behaviour - args = [node, edge, face] - state = not any(set(filter(lambda arg: arg is not None, args))) - node, edge, face = map( - lambda arg: arg if arg is not None else state, args - ) - - def func(args): - return args[1] is not None - - members = {} - if node: - members.update( - dict(filter(func, self.node_coords._asdict().items())) - ) - if edge: - members.update( - dict(filter(func, self.edge_coords._asdict().items())) - ) - if hasattr(self, "face_coords"): - if face: - members.update( - dict(filter(func, self.face_coords._asdict().items())) - ) - else: - dmsg = "Ignoring request to filter non-existent 'face_coords'" - logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) - - result = self._filters( - members, - item=item, - standard_name=standard_name, - long_name=long_name, - var_name=var_name, - attributes=attributes, - axis=axis, - ) - - return result - - def remove( - self, - item=None, - standard_name=None, - long_name=None, - var_name=None, - attributes=None, - axis=None, - node=None, - edge=None, - ): - return self._remove( - item=item, - standard_name=standard_name, - long_name=long_name, - var_name=var_name, - attributes=attributes, - axis=axis, - node=node, - edge=edge, - ) - - -class _Mesh2DCoordinateManager(_Mesh1DCoordinateManager): - OPTIONAL = ( - "edge_x", - "edge_y", - "face_x", - "face_y", - ) - - def __init__( - self, - node_x, - node_y, - edge_x=None, - edge_y=None, - face_x=None, - face_y=None, - ): - super().__init__(node_x, node_y, edge_x=edge_x, edge_y=edge_y) - - # optional coordinates - self.face_x = face_x - self.face_y = face_y + def face_node(self): + return self._members["face_node_connectivity"] @property - def _face_shape(self): - return self._shape(location="face") + def edge_node(self): + return self._members.get("edge_node_connectivity") @property - def all_members(self): - return Mesh2DCoords(**self._members) + def face_edge(self): + return self._members.get("face_edge_connectivity") @property - def face_coords(self): - return MeshFaceCoords(face_x=self.face_x, face_y=self.face_y) + def face_face(self): + return self._members.get("face_face_connectivity") @property - def face_x(self): - return self._members["face_x"] - - @face_x.setter - def face_x(self, coord): - self._setter( - location="face", axis="x", coord=coord, shape=self._face_shape - ) + def edge_face(self): + return self._members.get("edge_face_connectivity") @property - def face_y(self): - return self._members["face_y"] - - @face_y.setter - def face_y(self, coord): - self._setter( - location="face", axis="y", coord=coord, shape=self._face_shape - ) + def boundary_node(self): + return self._members.get("boundary_node_connectivity") - def add( - self, - node_x=None, - node_y=None, - edge_x=None, - edge_y=None, - face_x=None, - face_y=None, - ): - super().add(node_x=node_x, node_y=node_y, edge_x=edge_x, edge_y=edge_y) - self._add(MeshFaceCoords(face_x, face_y)) - def remove( - self, - item=None, - standard_name=None, - long_name=None, - var_name=None, - attributes=None, - axis=None, - node=None, - edge=None, - face=None, - ): - return self._remove( - item=item, - standard_name=standard_name, - long_name=long_name, - var_name=var_name, - attributes=attributes, - axis=axis, - node=node, - edge=edge, - face=face, - ) - - -# # keep an eye on the __init__ inheritance -# class _Mesh1DConnectivityManager: +# class Mesh(CFVariableMixin): +# """ +# +# .. todo:: +# +# .. questions:: +# +# - decide on the verbose/succinct version of __str__ vs __repr__ +# +# .. notes:: +# +# - the mesh is location agnostic +# +# - no need to support volume at mesh level, yet +# +# - topology_dimension +# - use for fast equality between Mesh instances +# - checking connectivity dimensionality, specifically the highest dimensonality of the +# "geometric element" being added i.e., reference the src_location/tgt_location +# - used to honour and enforce the minimum UGRID connectivity contract +# +# - support pickling +# +# - copy is off the table!! +# +# - MeshCoord.guess_points() +# - MeshCoord.to_AuxCoord() +# +# - don't provide public methods to return the coordinate and connectivity +# managers +# +# """ +# def __init__( +# self, +# topology_dimension, +# standard_name=None, +# long_name=None, +# var_name=None, +# units=None, +# attributes=None, +# node_dimension=None, +# edge_dimension=None, +# face_dimension=None, +# node_coords_and_axes=None, # [(coord, "x"), (coord, "y")] this is a stronger contract, not relying on guessing +# edge_coords_and_axes=None, # ditto +# face_coords_and_axes=None, # ditto +# connectivities=None, # [Connectivity, [Connectivity], ...] +# ): +# # TODO: support volumes. +# # TODO: support (coord, "z") +# +# # These are strings, if None is provided then assign the default string. +# self.node_dimension = node_dimension +# self.edge_dimension = edge_dimension +# self.face_dimension = face_dimension +# +# self._metadata_manager = metadata_manager_factory(MeshMetadata) +# +# self._metadata_manager.topology_dimension = topology_dimension +# +# self.standard_name = standard_name +# self.long_name = long_name +# self.var_name = var_name +# self.units = units +# self.attributes = attributes +# +# # based on the topology_dimension create the appropriate coordinate manager +# # with some intelligence +# self._coord_manager = ... +# +# # based on the topology_dimension create the appropriate connectivity manager +# # with some intelligence +# self._connectivity_manager = ... +# +# @property +# def all_coords(self): +# # return a namedtuple +# # coords = mesh.all_coords +# # coords.face_x, coords.edge_y +# pass +# +# @property +# def node_coords(self): +# # return a namedtuple +# # node_coords = mesh.node_coords +# # node_coords.x +# # node_coords.y +# pass +# +# @property +# def edge_coords(self): +# # as above +# pass +# +# @property +# def face_coords(self): +# # as above +# pass +# +# @property +# def all_connectivities(self): +# # return a namedtuple +# # conns = mesh.all_connectivities +# # conns.edge_node, conns.boundary_node +# pass +# +# @property +# def face_node_connectivity(self): +# # required +# return self._connectivity_manager.face_node +# +# @property +# def edge_node_connectivity(self): +# # optionally required +# return self._connectivity_manager.edge_node +# +# @property +# def face_edge_connectivity(self): +# # optional +# return self._connectivity_manager.face_edge +# +# @property +# def face_face_connectivity(self): +# # optional +# return self._connectivity_manager.face_face +# +# @property +# def edge_face_connectivity(self): +# # optional +# return self._connectivity_manager.edge_face +# +# @property +# def boundary_node_connectivity(self): +# # optional +# return self._connectivity_manager.boundard_node +# +# def coord(self, ...): +# # as Cube.coord i.e., ensure that one and only one coord-like is returned +# # otherwise raise and exception +# pass +# +# def coords( +# self, +# name_or_coord=None, +# standard_name=None, +# long_name=None, +# var_name=None, +# attributes=None, +# axis=None, +# node=False, +# edge=False, +# face=False, +# ): +# # do we support the coord_system kwargs? +# self._coord_manager.coords(...) +# +# def connectivity(self, ...): +# pass +# +# def connectivities( +# self, +# name_or_coord=None, +# standard_name=None, +# long_name=None, +# var_name=None, +# attributes=None, +# node=False, +# edge=False, +# face=False, +# ): +# pass +# +# def add_coords(self, node_x=None, node_y=None, edge_x=None, edge_y=None, face_x=None, face_y=None): +# # this supports adding a new coord to the manager, but also replacing an existing coord +# self._coord_manager.add(...) +# +# def add_connectivities(self, *args): +# # this supports adding a new connectivity to the manager, but also replacing an existing connectivity +# self._connectivity_manager.add(*args) +# +# def remove_coords(self, ...): +# # could provide the "name", "metadata", "coord"-instance +# # this could use mesh.coords() to find the coords +# self._coord_manager.remove(...) +# +# def remove_connectivities(self, ...): +# # needs to respect the minimum UGRID contract +# self._connectivity_manager.remove(...) +# +# def __eq__(self, other): +# # Full equality could be MASSIVE, so we want to avoid that. +# # Ideally we want a mesh signature from LFRic for comparison, although this would +# # limit Iris' relevance outside MO. +# # TL;DR: unknown quantity. +# raise NotImplemented +# +# def __ne__(self, other): +# # See __eq__ +# raise NotImplemented +# +# def __str__(self): +# pass +# +# def __repr__(self): +# pass +# +# def __unicode__(self, ...): +# pass +# +# def __getstate__(self): +# pass +# +# def __setstate__(self, state): +# pass +# +# def xml_element(self): +# pass +# +# # the MeshCoord will always have bounds, perhaps points. However the MeshCoord.guess_points() may +# # be a very useful part of its behaviour. +# # after using MeshCoord.guess_points(), the user may wish to add the associated MeshCoord.points into +# # the Mesh as face_coordinates. +# +# def to_AuxCoord(self, location, axis): +# # factory method +# # return the lazy AuxCoord(...) for the given location and axis +# +# def to_AuxCoords(self, location): +# # factory method +# # return the lazy AuxCoord(...), AuxCoord(...) +# +# def to_MeshCoord(self, location, axis): +# # factory method +# # return MeshCoord(..., location=location, axis=axis) +# # use Connectivity.indices_by_src() for fetching indices. +# +# def to_MeshCoords(self, location): +# # factory method +# # return MeshCoord(..., location=location, axis="x"), MeshCoord(..., location=location, axis="y") +# # use Connectivity.indices_by_src() for fetching indices. +# +# def dimension_names_reset(self, node=False, face=False, edge=False): +# # reset to defaults like this (suggestion) +# +# def dimension_names(self, node=None, face=None, edge=None): +# # e.g., only set self.node iff node != None. these attributes will +# # always be set to a user provided string or the default string. +# # return a namedtuple of dict-like +# +# @property +# def cf_role(self): +# return "mesh_topology" +# +# @property +# def topology_dimension(self): +# """ +# read-only +# +# """ +# return self._metadata_manager.topology_dimension +# +# # +# # - validate coord_systems +# # - validate climatological +# # - use guess_coord_axis (iris.utils) +# # - others? +# # +# class _Mesh1DCoordinateManager: # REQUIRED = ( -# "edge_node", +# "node_x", +# "node_y", # ) -# OPTIONAL = () -# def __init__(self, edge_node): +# OPTIONAL = ( +# "edge_x", +# "edge_y", +# ) +# def __init__(self, node_x, node_y, edge_x=None, edge_y=None): # # required -# self.edge_node = edge_node +# self.node_x = node_x +# self.node_y = node_y +# # optional +# self.edge_x = edge_x +# self.edge_y = edge_y # # # WOO-GA - this can easily get out of sync with the self attributes. # # choose the container wisely e.g., could be an dict..., also the self # # attributes may need to be @property's that access the chosen _members container -# -# # is this a list? as dict? a namedtuple? use case is self.add() -# self._members = [] -# -# if self.edge_node is not None: -# self._members.append(self.edge_node) +# self._members = [ ... ] # # def __iter__(self): # for member in self._members: @@ -1710,18 +1323,16 @@ def remove( # def __setstate__(self, state): # pass # -# def connectivity(self, **kwargs): +# def coord(self, **kwargs): # # see Cube.coord for pattern, checking for a single result -# return self.connectivities(**kwargs)[0] +# return self.coords(**kwargs)[0] # -# def connectivities(self, ...): +# def coords(self, ...): # # see Cube.coords for relevant patterns # # return [ ... ] # pass # -# def add(self, *args): -# # loop thru args and add (clobber) -# # adopt same philosophy as remove for adding connectivites with unsupported cf-role +# def add(self, **kwargs): # pass # # def remove(self, ...): @@ -1747,56 +1358,37 @@ def remove( # raise NotImplemented # # -# class _Mesh2DConnectivityManager(_Mesh1DConnectivityManager): -# REQUIRED = ( -# "face_node", -# ) +# class _Mesh2DCoordinateManager(_Mesh1DCoordinateManager): # OPTIONAL = ( -# "edge_node", -# "face_edge", -# "face_face", -# "edge_face", -# "boundary_node", +# "edge_x", +# "edge_y", +# "face_x", +# "face_y", # ) -# def __init__(self, face_node, edge_node=None, face_edge=None, face_face=None, edge_face=None, boundary_node=None): -# # required -# self.face_node = face_node -# self._members = [self.face_node] -# -# # optionally required -# self.edge_node = edge_node +# def __init__(self, node_x, node_y, edge_x=None, edge_y=None, face_x=None, face_y=None): # # optional -# self.face_edge = face_edge -# self.face_face = face_face -# self.edge_face = edge_face -# self.boundary_node = boundary_node +# self.face_x = face_x +# self.face_y = face_y +# +# super().__init__(node_x, node_y, edge_x=edge_x, edge_y=edge_y) +# +# # does the order matter? +# self._members.extend([self.face_x, self.face_y]) # -# # edge_node could be None here. are we okay with this pattern? -# super().__init__(edge_node) # -# # does order matter? -# self._members.extend([member for member in self.OPTIONAL if member is not None and member != "edge_node"]) #: Convenience collection of lenient metadata combine services. -_services = [ConnectivityMetadata.combine, MeshMetadata.combine] -SERVICES_COMBINE.extend(_services) -SERVICES.extend(_services) +SERVICES_COMBINE.append(ConnectivityMetadata.combine) +SERVICES.append(ConnectivityMetadata.combine) #: Convenience collection of lenient metadata difference services. -_services = [ConnectivityMetadata.difference, MeshMetadata.difference] -SERVICES_DIFFERENCE.extend(_services) -SERVICES.extend(_services) +SERVICES_DIFFERENCE.append(ConnectivityMetadata.difference) +SERVICES.append(ConnectivityMetadata.difference) #: Convenience collection of lenient metadata equality services. -_services = [ - ConnectivityMetadata.__eq__, - ConnectivityMetadata.equal, - MeshMetadata.__eq__, - MeshMetadata.equal, -] -SERVICES_EQUAL.extend(_services) -SERVICES.extend(_services) - -del _services +SERVICES_EQUAL.extend( + [ConnectivityMetadata.__eq__, ConnectivityMetadata.equal] +) +SERVICES.extend([ConnectivityMetadata.__eq__, ConnectivityMetadata.equal]) From f3d109edf5b22b17db8d8790b55e22439903be24 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Thu, 18 Feb 2021 14:30:34 +0000 Subject: [PATCH 02/22] ConnectivityManager align with proposed CoordManager. --- lib/iris/common/__init__.py | 4 +- lib/iris/experimental/ugrid.py | 116 ++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 54 deletions(-) diff --git a/lib/iris/common/__init__.py b/lib/iris/common/__init__.py index 097c969904..211c07a564 100644 --- a/lib/iris/common/__init__.py +++ b/lib/iris/common/__init__.py @@ -68,7 +68,9 @@ def filter_cf( def attr_filter(instance_): return all( - k in instance_.attributes and instance_.attributes[k] == v + k in instance_.attributes + and metadata._hexdigest(instance_.attributes[k]) + == metadata._hexdigest(v) for k, v in attributes.items() ) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 129b69cb17..da4f77c43a 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -13,7 +13,6 @@ from abc import ABC, abstractmethod from collections import namedtuple from functools import wraps -from warnings import warn import dask.array as da import numpy as np @@ -29,6 +28,7 @@ SERVICES_DIFFERENCE, ) from ..common.lenient import _lenient_service as lenient_service +from ..config import get_logger from ..coords import _DimensionalMetadata from .. import exceptions @@ -36,9 +36,28 @@ __all__ = [ "Connectivity", "ConnectivityMetadata", + "Mesh1DConnectivities", + "Mesh2DConnectivities", ] +# Configure the logger. +logger = get_logger(__name__, fmt="[%(cls)s.%(funcName)s]") + +Mesh1DConnectivities = namedtuple("Mesh1DConnectivities", ["edge_node"]) +Mesh2DConnectivities = namedtuple( + "Mesh2DConnectivities", + [ + "face_node", + "edge_node", + "face_edge", + "face_face", + "edge_face", + "boundary_node", + ], +) + + class Connectivity(_DimensionalMetadata): """ A CF-UGRID topology connectivity, describing the topological relationship @@ -745,7 +764,6 @@ class _MeshConnectivityManagerMixin(ABC): REQUIRED = () OPTIONAL = () NDIM = NotImplemented - MembersTuple = NotImplemented @abstractmethod def __init__(self, *connectivities): @@ -755,18 +773,20 @@ def __init__(self, *connectivities): message = f"{self.NDIM}D meshes require a {requisite}." raise ValueError(message) - self.valid_roles = self.REQUIRED + self.OPTIONAL + self.ALL = self.REQUIRED + self.OPTIONAL self._members = {} self.add(*connectivities) def __iter__(self): - for member, connectivity in self._members.items(): - yield member, connectivity + for item in self._members.items(): + yield item def __getstate__(self): + # TBD pass def __setstate__(self, state): + # TBD pass @property @@ -785,7 +805,6 @@ def filter( edge=None, face=None, ): - # see Cube.coords for relevant patterns result = filter_cf( self._members.values(), item=item, @@ -795,24 +814,26 @@ def filter( attributes=attributes, ) - def location_filter(instances, parameter, location_name): - result = instances - if parameter is True: - result = [ + def location_filter(instances_, parameter_, location_name_): + if parameter_ is False: + result_ = [ instance_ - for instance_ in result - if location_name - in (instance_.src_location, instance_.tgt_location) + for instance_ in instances_ + if location_name_ + not in (instance_.src_location, instance_.tgt_location) ] - elif parameter is False: - result = [ + elif parameter_ is None: + result_ = instances_ + else: + # Interpret any other value as =True. + result_ = [ instance_ - for instance_ in result - if location_name - not in (instance_.src_location, instance_.tgt_location) + for instance_ in instances_ + if location_name_ + in (instance_.src_location, instance_.tgt_location) ] - return result + return result_ for parameter, location_name in ( (node, "node"), @@ -821,7 +842,8 @@ def location_filter(instances, parameter, location_name): ): result = location_filter(result, parameter, location_name) - return result + result_dict = {k: v for k, v in self._members.items() if v in result} + return result_dict def filter_single(self, **kwargs): result = self.filter(**kwargs) @@ -856,14 +878,18 @@ def add(self, *connectivities): # validate their outputs. add_dict = {} for connectivity in connectivities: - assert isinstance(connectivity, Connectivity) + if not isinstance(connectivity, Connectivity): + message = f"Expected Connectivity, got: {type(connectivity)} ." + raise ValueError(message) cf_role = connectivity.cf_role - if cf_role not in self.valid_roles: + if cf_role not in self.ALL: message = ( f"Connectivity not added. Got cf_role={cf_role} . " - f"Expected one of: {self.valid_roles} ." + f"Expected one of: {self.ALL} ." + ) + logger.warning( + message, extra=dict(cls=self.__class__.__name__) ) - warn(message) else: add_dict[cf_role] = connectivity @@ -916,7 +942,9 @@ def remove( f"Connectivity not removed: {cf_role} - required " f"for a valid {self.NDIM}D Mesh." ) - warn(message) + logger.warning( + message, extra=dict(cls=self.__class__.__name__) + ) for cf_role in removal_dict.keys(): del self._members[cf_role] @@ -924,22 +952,16 @@ def remove( return removal_dict def __repr__(self): - class_name = type(self).__name__ - content = ", ".join( - f"{member}={connectivity}" for member, connectivity in self - ) - return ", ".join((class_name, content)) + args = [f"{member}={connectivity}" for member, connectivity in self] + return f"{self.__class__.__name__}({', '.join(args)})" def __eq__(self, other): - # Full equality could be MASSIVE, so we want to avoid that. - # Ideally we want a mesh signature from LFRic for comparison, although this would - # limit Iris' relevance outside MO. - # TL;DR: unknown quantity. - raise NotImplementedError + # TBD + return NotImplemented def __ne__(self, other): - # See __eq__ - raise NotImplementedError + # TBD + return NotImplemented # keep an eye on the __init__ inheritance @@ -947,15 +969,14 @@ class _Mesh1DConnectivityManager(_MeshConnectivityManagerMixin): REQUIRED = ("edge_node_connectivity",) OPTIONAL = () NDIM = 1 - MembersTuple = namedtuple("Mesh1DConnectivities", ["edge_node"]) def __init__(self, *connectivities): super().__init__(*connectivities) - # TODO: debatable whether one couldn't just use self.filter(). + # TODO: debatable whether a user couldn't just use self.filter() with no args. @property def all_members(self): - return self.MembersTuple(self.edge_node) + return Mesh1DConnectivities(self.edge_node) @property def edge_node(self): @@ -972,25 +993,14 @@ class _Mesh2DConnectivityManager(_MeshConnectivityManagerMixin): "boundary_node_connectivity", ) NDIM = 2 - MembersTuple = namedtuple( - "Mesh2DConnectivities", - [ - "face_node", - "edge_node", - "face_edge", - "face_face", - "edge_face", - "boundary_node", - ], - ) def __init__(self, *connectivities): super().__init__(*connectivities) - # TODO: debatable whether one couldn't just use self.filter(). + # TODO: debatable whether a user couldn't just use self.filter() with no args. @property def all_members(self): - return self.MembersTuple( + return Mesh2DConnectivities( self.face_node, self.edge_node, self.face_edge, From 5a46ea1ba05d38039c54057cf92327e6851fd535 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Thu, 18 Feb 2021 17:01:27 +0000 Subject: [PATCH 03/22] Connectivity Manager review actions. --- lib/iris/experimental/ugrid.py | 205 +++++++++++++++++---------------- 1 file changed, 103 insertions(+), 102 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index da4f77c43a..9241c83404 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -761,6 +761,7 @@ def equal(self, other, lenient=None): class _MeshConnectivityManagerMixin(ABC): + # TODO: re-order methods, properties etc to align with Coord Managers. REQUIRED = () OPTIONAL = () NDIM = NotImplemented @@ -777,24 +778,109 @@ def __init__(self, *connectivities): self._members = {} self.add(*connectivities) + def __eq__(self, other): + # TBD + return NotImplemented + + def __getstate__(self): + # TBD + pass + def __iter__(self): for item in self._members.items(): yield item - def __getstate__(self): + def __ne__(self, other): # TBD - pass + return NotImplemented + + def __repr__(self): + args = [f"{member}={connectivity!r}" for member, connectivity in self] + return f"{self.__class__.__name__}({', '.join(args)})" def __setstate__(self, state): # TBD pass + def __str__(self): + args = [f"{member}=True" for member, connectivity in self] + return f"{self.__class__.__name__}({', '.join(args)})" + @property @abstractmethod def all_members(self): return NotImplemented - def filter( + def add(self, *connectivities): + # Since Connectivity classes include their cf_role, no setters will be + # provided, just a means to add one or more connectivities to the + # manager. + # No warning is raised for duplicate cf_roles - user is trusted to + # validate their outputs. + add_dict = {} + for connectivity in connectivities: + if not isinstance(connectivity, Connectivity): + message = f"Expected Connectivity, got: {type(connectivity)} ." + raise ValueError(message) + cf_role = connectivity.cf_role + if cf_role not in self.ALL: + message = ( + f"Not adding connectivity {connectivity!r} - cf_role must " + f"be one of: {self.ALL} ." + ) + logger.debug(message, extra=dict(cls=self.__class__.__name__)) + else: + add_dict[cf_role] = connectivity + + # Validate shapes. + proposed_members = {**self._members, **add_dict} + locations = set([c.src_location for c in proposed_members.values()]) + for location in locations: + counts = [ + len(c.indices_by_src()) + for c in proposed_members.values() + if c.src_location == location + ] + # Check is list values are identical. + if not counts.count(counts[0]) == len(counts): + message = ( + f"Invalid Connectivities provided - inconsistent " + f"{location} counts." + ) + raise ValueError(message) + + self._members = proposed_members + + def filter(self, **kwargs): + result = self.filters(**kwargs) + if len(result) > 1: + names = ", ".join( + f"{member}={connectivity!r}" + for member, connectivity in result.items() + ) + message = ( + f"Expected to find exactly 1 connectivity, but found " + f"{len(result)}. They were: {names}." + ) + raise exceptions.ConnectivityNotFoundError(message) + elif len(result) == 0: + item = kwargs["item"] + _name = item + if item is not None: + if not isinstance(item, str): + _name = item.name() + bad_name = ( + _name or kwargs["standard_name"] or kwargs["long_name"] or "" + ) + message = ( + f"Expected to find exactly 1 {bad_name} connectivity, " + f"but found none." + ) + raise exceptions.ConnectivityNotFoundError(message) + + return result + + def filters( self, item=None, standard_name=None, @@ -845,73 +931,6 @@ def location_filter(instances_, parameter_, location_name_): result_dict = {k: v for k, v in self._members.items() if v in result} return result_dict - def filter_single(self, **kwargs): - result = self.filter(**kwargs) - if len(result) > 1: - message = ( - f"Expected to find exactly 1 connectivity, but found " - f"{len(result)}. They were: {[c.name for c in result]}." - ) - raise exceptions.ConnectivityNotFoundError(message) - elif len(result) == 0: - item = kwargs["item"] - _name = item - if item is not None: - if not isinstance(item, str): - _name = item.name() - bad_name = ( - _name or kwargs["standard_name"] or kwargs["long_name"] or "" - ) - message = ( - f"Expected to find exactly 1 {bad_name} connectivity, " - f"but found none." - ) - raise exceptions.ConnectivityNotFoundError(message) - - return result[0] - - def add(self, *connectivities): - # Since Connectivity classes include their cf_role, no setters will be - # provided, just a means to add one or more connectivities to the - # manager. - # No warning is raised for duplicate cf_roles - user is trusted to - # validate their outputs. - add_dict = {} - for connectivity in connectivities: - if not isinstance(connectivity, Connectivity): - message = f"Expected Connectivity, got: {type(connectivity)} ." - raise ValueError(message) - cf_role = connectivity.cf_role - if cf_role not in self.ALL: - message = ( - f"Connectivity not added. Got cf_role={cf_role} . " - f"Expected one of: {self.ALL} ." - ) - logger.warning( - message, extra=dict(cls=self.__class__.__name__) - ) - else: - add_dict[cf_role] = connectivity - - # Validate shapes. - proposed_members = {**self._members, **add_dict} - locations = set([c.src_location for c in proposed_members.values()]) - for location in locations: - counts = [ - len(c.indices_by_src()) - for c in proposed_members.values() - if c.src_location == location - ] - # Check is list values are identical. - if not counts.count(counts[0]) == len(counts): - message = ( - f"Invalid Connectivities provided - inconsistent " - f"{location} counts." - ) - raise ValueError(message) - - self._members = proposed_members - def remove( self, item=None, @@ -923,9 +942,7 @@ def remove( edge=None, face=None, ): - # use logging/warning to flag items not removed - highlight in doc-string - # don't raise an exception - removal_list = self.filter( + removal_list = self.filters( item=item, standard_name=standard_name, long_name=long_name, @@ -937,34 +954,20 @@ def remove( ) removal_dict = {c.cf_role: c for c in removal_list} for cf_role in self.REQUIRED: - if removal_dict.pop(cf_role, None): + not_removed = removal_dict.pop(cf_role, None) + if not_removed: message = ( - f"Connectivity not removed: {cf_role} - required " - f"for a valid {self.NDIM}D Mesh." - ) - logger.warning( - message, extra=dict(cls=self.__class__.__name__) + f"Ignoring request to remove required connectivity " + f"{not_removed!r}" ) + logger.debug(message, extra=dict(cls=self.__class__.__name__)) for cf_role in removal_dict.keys(): del self._members[cf_role] return removal_dict - def __repr__(self): - args = [f"{member}={connectivity}" for member, connectivity in self] - return f"{self.__class__.__name__}({', '.join(args)})" - - def __eq__(self, other): - # TBD - return NotImplemented - def __ne__(self, other): - # TBD - return NotImplemented - - -# keep an eye on the __init__ inheritance class _Mesh1DConnectivityManager(_MeshConnectivityManagerMixin): REQUIRED = ("edge_node_connectivity",) OPTIONAL = () @@ -973,7 +976,6 @@ class _Mesh1DConnectivityManager(_MeshConnectivityManagerMixin): def __init__(self, *connectivities): super().__init__(*connectivities) - # TODO: debatable whether a user couldn't just use self.filter() with no args. @property def all_members(self): return Mesh1DConnectivities(self.edge_node) @@ -997,7 +999,6 @@ class _Mesh2DConnectivityManager(_MeshConnectivityManagerMixin): def __init__(self, *connectivities): super().__init__(*connectivities) - # TODO: debatable whether a user couldn't just use self.filter() with no args. @property def all_members(self): return Mesh2DConnectivities( @@ -1010,8 +1011,12 @@ def all_members(self): ) @property - def face_node(self): - return self._members["face_node_connectivity"] + def boundary_node(self): + return self._members.get("boundary_node_connectivity") + + @property + def edge_face(self): + return self._members.get("edge_face_connectivity") @property def edge_node(self): @@ -1026,12 +1031,8 @@ def face_face(self): return self._members.get("face_face_connectivity") @property - def edge_face(self): - return self._members.get("edge_face_connectivity") - - @property - def boundary_node(self): - return self._members.get("boundary_node_connectivity") + def face_node(self): + return self._members["face_node_connectivity"] # class Mesh(CFVariableMixin): From 91a3ed60232e92bd90db6fb78365e9acdccc6733 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 19 Feb 2021 14:03:07 +0000 Subject: [PATCH 04/22] Connectivity Manager more review changes. --- lib/iris/experimental/ugrid.py | 94 ++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 32 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 9241c83404..847b198a06 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -761,7 +761,6 @@ def equal(self, other, lenient=None): class _MeshConnectivityManagerMixin(ABC): - # TODO: re-order methods, properties etc to align with Coord Managers. REQUIRED = () OPTIONAL = () NDIM = NotImplemented @@ -775,7 +774,7 @@ def __init__(self, *connectivities): raise ValueError(message) self.ALL = self.REQUIRED + self.OPTIONAL - self._members = {} + self._members = {member: None for member in self.ALL} self.add(*connectivities) def __eq__(self, other): @@ -795,7 +794,11 @@ def __ne__(self, other): return NotImplemented def __repr__(self): - args = [f"{member}={connectivity!r}" for member, connectivity in self] + args = [ + f"{member}={connectivity!r}" + for member, connectivity in self + if connectivity is not None + ] return f"{self.__class__.__name__}({', '.join(args)})" def __setstate__(self, state): @@ -803,7 +806,11 @@ def __setstate__(self, state): pass def __str__(self): - args = [f"{member}=True" for member, connectivity in self] + args = [ + f"{member}=True" + for member, connectivity in self + if connectivity is not None + ] return f"{self.__class__.__name__}({', '.join(args)})" @property @@ -825,8 +832,8 @@ def add(self, *connectivities): cf_role = connectivity.cf_role if cf_role not in self.ALL: message = ( - f"Not adding connectivity {connectivity!r} - cf_role must " - f"be one of: {self.ALL} ." + f"Not adding connectivity ({cf_role}: " + f"{connectivity!r}) - cf_role must be one of: {self.ALL} ." ) logger.debug(message, extra=dict(cls=self.__class__.__name__)) else: @@ -834,12 +841,18 @@ def add(self, *connectivities): # Validate shapes. proposed_members = {**self._members, **add_dict} - locations = set([c.src_location for c in proposed_members.values()]) + locations = set( + [ + c.src_location + for c in proposed_members.values() + if c is not None + ] + ) for location in locations: counts = [ len(c.indices_by_src()) for c in proposed_members.values() - if c.src_location == location + if c is not None and c.src_location == location ] # Check is list values are identical. if not counts.count(counts[0]) == len(counts): @@ -887,12 +900,13 @@ def filters( long_name=None, var_name=None, attributes=None, + cf_role=None, node=None, edge=None, face=None, ): - result = filter_cf( - self._members.values(), + members = filter_cf( + [c for c in self._members.values() if c is not None], item=item, standard_name=standard_name, long_name=long_name, @@ -900,35 +914,50 @@ def filters( attributes=attributes, ) + if cf_role is not None: + members = [ + instance_ + for instance_ in members + if instance_.cf_role == cf_role + ] + def location_filter(instances_, parameter_, location_name_): if parameter_ is False: - result_ = [ + members_ = [ instance_ for instance_ in instances_ if location_name_ not in (instance_.src_location, instance_.tgt_location) ] elif parameter_ is None: - result_ = instances_ + members_ = instances_ else: # Interpret any other value as =True. - result_ = [ + members_ = [ instance_ for instance_ in instances_ if location_name_ in (instance_.src_location, instance_.tgt_location) ] - return result_ + return members_ for parameter, location_name in ( (node, "node"), (edge, "edge"), (face, "face"), ): - result = location_filter(result, parameter, location_name) + members = location_filter(members, parameter, location_name) - result_dict = {k: v for k, v in self._members.items() if v in result} + # No need to actually modify filtering behaviour - already won't return + # any face cf-roles if none are present. + if self.NDIM < 2: + message = ( + "Ignoring request to filter for non-existent 'face' cf-roles." + ) + logger.debug(message, extra=dict(cls=self.__class__.__name__)) + + result_dict = {k: v for k, v in self._members.items() if v in members} return result_dict def remove( @@ -938,32 +967,33 @@ def remove( long_name=None, var_name=None, attributes=None, + cf_role=None, node=None, edge=None, face=None, ): - removal_list = self.filters( + removal_dict = self.filters( item=item, standard_name=standard_name, long_name=long_name, var_name=var_name, attributes=attributes, + cf_role=cf_role, node=node, edge=edge, face=face, ) - removal_dict = {c.cf_role: c for c in removal_list} for cf_role in self.REQUIRED: not_removed = removal_dict.pop(cf_role, None) if not_removed: message = ( f"Ignoring request to remove required connectivity " - f"{not_removed!r}" + f"({cf_role}: {not_removed!r})" ) logger.debug(message, extra=dict(cls=self.__class__.__name__)) for cf_role in removal_dict.keys(): - del self._members[cf_role] + self._members[cf_role] = None return removal_dict @@ -978,7 +1008,7 @@ def __init__(self, *connectivities): @property def all_members(self): - return Mesh1DConnectivities(self.edge_node) + return Mesh1DConnectivities(edge_node=self.edge_node) @property def edge_node(self): @@ -1002,33 +1032,33 @@ def __init__(self, *connectivities): @property def all_members(self): return Mesh2DConnectivities( - self.face_node, - self.edge_node, - self.face_edge, - self.face_face, - self.edge_face, - self.boundary_node, + face_node=self.face_node, + edge_node=self.edge_node, + face_edge=self.face_edge, + face_face=self.face_face, + edge_face=self.edge_face, + boundary_node=self.boundary_node, ) @property def boundary_node(self): - return self._members.get("boundary_node_connectivity") + return self._members["boundary_node_connectivity"] @property def edge_face(self): - return self._members.get("edge_face_connectivity") + return self._members["edge_face_connectivity"] @property def edge_node(self): - return self._members.get("edge_node_connectivity") + return self._members["edge_node_connectivity"] @property def face_edge(self): - return self._members.get("face_edge_connectivity") + return self._members["face_edge_connectivity"] @property def face_face(self): - return self._members.get("face_face_connectivity") + return self._members["face_face_connectivity"] @property def face_node(self): From 487859f2b8028943b9bd6d0d5c81d6246891ff4f Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 19 Feb 2021 14:23:45 +0000 Subject: [PATCH 05/22] Use metadata_manager for Mesh location dimension. --- lib/iris/experimental/ugrid.py | 1323 +++++++++++++++++++++++--------- 1 file changed, 959 insertions(+), 364 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 847b198a06..9722d2e7e9 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -11,7 +11,7 @@ """ from abc import ABC, abstractmethod -from collections import namedtuple +from collections import Mapping, namedtuple from functools import wraps import dask.array as da @@ -20,7 +20,10 @@ from .. import _lazy_data as _lazy from ..common import filter_cf from ..common.metadata import ( + _hexdigest, BaseMetadata, + CoordMetadata, + DimCoordMetadata, metadata_manager_factory, SERVICES, SERVICES_COMBINE, @@ -28,8 +31,11 @@ SERVICES_DIFFERENCE, ) from ..common.lenient import _lenient_service as lenient_service +from ..common.mixin import CFVariableMixin from ..config import get_logger -from ..coords import _DimensionalMetadata +from ..coords import _DimensionalMetadata, AuxCoord +from ..exceptions import CoordinateNotFoundError +from ..util import guess_coord_axis from .. import exceptions @@ -37,13 +43,38 @@ "Connectivity", "ConnectivityMetadata", "Mesh1DConnectivities", + "Mesh1DCoords", "Mesh2DConnectivities", + "Mesh2DCoords", + "MeshEdgeCoords", + "MeshFaceCoords", + "MeshNodeCoords", + "MeshMetadata", ] # Configure the logger. logger = get_logger(__name__, fmt="[%(cls)s.%(funcName)s]") + +# Mesh dimension names namedtuples. +Mesh1DNames = namedtuple("Mesh1DNames", ["node_dimension", "edge_dimension"]) +Mesh2DNames = namedtuple( + "Mesh2DNames", ["node_dimension", "edge_dimension", "face_dimension"] +) + +# Mesh coordinate manager namedtuples. +Mesh1DCoords = namedtuple( + "Mesh1DCoords", ["node_x", "node_y", "edge_x", "edge_y"] +) +Mesh2DCoords = namedtuple( + "Mesh2DCoords", + ["node_x", "node_y", "edge_x", "edge_y", "face_x", "face_y"], +) +MeshNodeCoords = namedtuple("MeshNodeCoords", ["node_x", "node_y"]) +MeshEdgeCoords = namedtuple("MeshEdgeCoords", ["edge_x", "edge_y"]) +MeshFaceCoords = namedtuple("MeshFaceCoords", ["face_x", "face_y"]) + Mesh1DConnectivities = namedtuple("Mesh1DConnectivities", ["edge_node"]) Mesh2DConnectivities = namedtuple( "Mesh2DConnectivities", @@ -760,6 +791,916 @@ def equal(self, other, lenient=None): return super().equal(other, lenient=lenient) +class Mesh(CFVariableMixin): + """ + + .. todo:: + + .. questions:: + + - decide on the verbose/succinct version of __str__ vs __repr__ + + .. notes:: + + - the mesh is location agnostic + + - no need to support volume at mesh level, yet + + - topology_dimension + - use for fast equality between Mesh instances + - checking connectivity dimensionality, specifically the highest dimensonality of the + "geometric element" being added i.e., reference the src_location/tgt_location + - used to honour and enforce the minimum UGRID connectivity contract + + - support pickling + + - copy is off the table!! + + - MeshCoord.guess_points() + - MeshCoord.to_AuxCoord() + + - don't provide public methods to return the coordinate and connectivity + managers + + - validate both managers contents e.g., shape? more...? + + """ + + # TBD: for volume and/or z-axis support include axis "z" and/or dimension "3" + AXES = ("x", "y") + TOPOLOGY_DIMENSIONS = (1, 2) + + def __init__( + self, + topology_dimension, + node_coords_and_axes, + standard_name=None, + long_name=None, + var_name=None, + units=None, + attributes=None, + edge_coords_and_axes=None, + face_coords_and_axes=None, + # connectivities=None, + node_dimension=None, + edge_dimension=None, + face_dimension=None, + ): + # TODO: support volumes. + # TODO: support (coord, "z") + + self._metadata_manager = metadata_manager_factory(MeshMetadata) + + # topology_dimension is read-only, so assign directly to the metadata manager + if topology_dimension not in self.TOPOLOGY_DIMENSIONS: + emsg = f"Expected 'topology_dimension' in range {self.TOPOLOGY_DIMENSIONS!r}, got {topology_dimension!r}." + raise ValueError(emsg) + self._metadata_manager.topology_dimension = topology_dimension + + # TBD: these are strings, if None is provided then assign the default string. + self.node_dimension = node_dimension + self.edge_dimension = edge_dimension + self.face_dimension = face_dimension + + # assign the metadata to the metadata manager + self.standard_name = standard_name + self.long_name = long_name + self.var_name = var_name + self.units = units + self.attributes = attributes + + # based on the topology_dimension, create the appropriate coordinate manager + def normalise(location, axis): + result = str(axis).lower() + if result not in self.AXES: + emsg = f"Invalid axis specified for {location} coordinate {coord.name()!r}, got {axis!r}." + raise ValueError(emsg) + return f"{location}_{axis}" + + kwargs = {} + for coord, axis in node_coords_and_axes: + kwargs[normalise("node", axis)] = coord + if edge_coords_and_axes is not None: + for coord, axis in edge_coords_and_axes: + kwargs[normalise("edge", axis)] = coord + if face_coords_and_axes is not None: + for coord, axis in face_coords_and_axes: + kwargs[normalise("face", axis)] = coord + + # check the UGRID minimum requirement for coordinates + if "node_x" not in kwargs: + emsg = ( + "Require a node coordinate that is x-axis like to be provided." + ) + raise ValueError(emsg) + if "node_y" not in kwargs: + emsg = ( + "Require a node coordinate that is y-axis like to be provided." + ) + raise ValueError(emsg) + + if self.topology_dimension == 1: + self._coord_manager = _Mesh1DCoordinateManager(**kwargs) + elif self.topology_dimension == 2: + self._coord_manager = _Mesh2DCoordinateManager(**kwargs) + else: + emsg = f"Unsupported 'topology_dimension', got {topology_dimension!r}." + raise NotImplementedError(emsg) + + # based on the topology_dimension, create the appropriate connectivity manager + # self._connectivity_manager = ... + + def __eq__(self, other): + # TBD + return NotImplemented + + def __getstate__(self): + # TBD + pass + + def __ne__(self, other): + # TBD + return NotImplemented + + def __repr__(self): + # TBD + args = [] + return f"{self.__class__.__name__}({', '.join(args)})" + + def __setstate__(self, state): + # TBD + pass + + def __str__(self): + # TBD + args = [] + return f"{self.__class__.__name__}({', '.join(args)})" + + @property + def all_coords(self): + return self._coord_manager.all_members + + @property + def edge_dimension(self): + return self._metadata_manager.edge_dimension + + @edge_dimension.setter + def edge_dimension(self, name): + if not name or not isinstance(name, str): + edge_dimension_ = f"Mesh{self.topology_dimension}d_edge" + else: + edge_dimension_ = name + self._metadata_manager.edge_dimension = edge_dimension_ + + @property + def edge_coords(self): + return self._coord_manager.edge_coords + + @property + def face_dimension(self): + return self._metadata_manager.face_dimension + + @face_dimension.setter + def face_dimension(self, name): + if not name or not isinstance(name, str): + face_dimension_ = f"Mesh{self.topology_dimension}d_face" + else: + face_dimension_ = name + self._metadata_manager.face_dimension = face_dimension_ + + @property + def face_coords(self): + return self._coord_manager.face_coords + + @property + def node_dimension(self): + return self._metadata_manager.node_dimension + + @node_dimension.setter + def node_dimension(self, name): + if not name or not isinstance(name, str): + node_dimension_ = f"Mesh{self.topology_dimension}d_node" + else: + node_dimension_ = name + self._metadata_manager.node_dimension = node_dimension_ + + @property + def node_coords(self): + return self._coord_manager.node_coords + + # @property + # def all_connectivities(self): + # # return a namedtuple + # # conns = mesh.all_connectivities + # # conns.edge_node, conns.boundary_node + # pass + # + # @property + # def face_node_connectivity(self): + # # required + # return self._connectivity_manager.face_node + # + # @property + # def edge_node_connectivity(self): + # # optionally required + # return self._connectivity_manager.edge_node + # + # @property + # def face_edge_connectivity(self): + # # optional + # return self._connectivity_manager.face_edge + # + # @property + # def face_face_connectivity(self): + # # optional + # return self._connectivity_manager.face_face + # + # @property + # def edge_face_connectivity(self): + # # optional + # return self._connectivity_manager.edge_face + # + # @property + # def boundary_node_connectivity(self): + # # optional + # return self._connectivity_manager.boundary_node + + def add_coords( + self, + node_x=None, + node_y=None, + edge_x=None, + edge_y=None, + face_x=None, + face_y=None, + ): + self._coord_manager.add( + node_x=node_x, + node_y=node_y, + edge_x=edge_x, + edge_y=edge_y, + face_x=face_x, + face_y=face_y, + ) + + # def add_connectivities(self, *args): + # # this supports adding a new connectivity to the manager, but also replacing an existing connectivity + # self._connectivity_manager.add(*args) + + # def connectivities( + # self, + # name_or_coord=None, + # standard_name=None, + # long_name=None, + # var_name=None, + # attributes=None, + # node=False, + # edge=False, + # face=False, + # ): + # pass + + # def connectivity(self, ...): + # pass + + def coord( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, + node=None, + edge=None, + face=None, + ): + return self._coord_manager.filter( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + axis=axis, + node=node, + edge=edge, + face=face, + ) + + def coords( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, + node=False, + edge=False, + face=False, + ): + return self._coord_manager.filters( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + axis=axis, + node=node, + edge=edge, + face=face, + ) + + # def remove_connectivities(self, ...): + # # needs to respect the minimum UGRID contract + # self._connectivity_manager.remove(...) + + def remove_coords( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, + node=None, + edge=None, + face=None, + ): + self._coord_manager.remove( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + axis=axis, + node=node, + edge=edge, + face=face, + ) + + def xml_element(self): + # TBD + pass + + # the MeshCoord will always have bounds, perhaps points. However the MeshCoord.guess_points() may + # be a very useful part of its behaviour. + # after using MeshCoord.guess_points(), the user may wish to add the associated MeshCoord.points into + # the Mesh as face_coordinates. + + # def to_AuxCoord(self, location, axis): + # # factory method + # # return the lazy AuxCoord(...) for the given location and axis + # + # def to_AuxCoords(self, location): + # # factory method + # # return the lazy AuxCoord(...), AuxCoord(...) + # + # def to_MeshCoord(self, location, axis): + # # factory method + # # return MeshCoord(..., location=location, axis=axis) + # # use Connectivity.indices_by_src() for fetching indices. + # + # def to_MeshCoords(self, location): + # # factory method + # # return MeshCoord(..., location=location, axis="x"), MeshCoord(..., location=location, axis="y") + # # use Connectivity.indices_by_src() for fetching indices. + + def dimension_names_reset(self, node=False, edge=False, face=False): + if node: + self.node_dimension = None + if edge: + self.edge_dimension = None + if face: + self.face_dimension = None + if self.topology_dimension == 1: + result = Mesh1DNames(self.node_dimension, self.edge_dimension) + else: + result = Mesh2DNames( + self.node_dimension, self.edge_dimension, self.face_dimension + ) + return result + + def dimension_names(self, node=None, edge=None, face=None): + if node: + self.node_dimension = node + if edge: + self.edge_dimension = edge + if face: + self.face_dimension = face + if self.topology_dimension == 1: + result = Mesh1DNames(self.node_dimension, self.edge_dimension) + else: + result = Mesh2DNames( + self.node_dimension, self.edge_dimension, self.node_dimension + ) + return result + + @property + def cf_role(self): + return "mesh_topology" + + @property + def topology_dimension(self): + return self._metadata_manager.topology_dimension + + +class _Mesh1DCoordinateManager: + """ + + TBD: require clarity on coord_systems validation + TBD: require clarity on __eq__ support + TBD: rationalise self.coords() logic with other manager and Cube + + """ + + REQUIRED = ( + "node_x", + "node_y", + ) + OPTIONAL = ( + "edge_x", + "edge_y", + ) + + def __init__(self, node_x, node_y, edge_x=None, edge_y=None): + # initialise all the coordinates + self.ALL = self.REQUIRED + self.OPTIONAL + self._members = {member: None for member in self.ALL} + + # required coordinates + self.node_x = node_x + self.node_y = node_y + # optional coordinates + self.edge_x = edge_x + self.edge_y = edge_y + + def __eq__(self, other): + # TBD + return NotImplemented + + def __getstate__(self): + # TBD + pass + + def __iter__(self): + for item in self._members.items(): + yield item + + def __ne__(self, other): + # TBD + return NotImplemented + + def __repr__(self): + args = [ + f"{member}={coord!r}" + for member, coord in self + if coord is not None + ] + return f"{self.__class__.__name__}({', '.join(args)})" + + def __setstate__(self, state): + # TBD + pass + + def __str__(self): + args = [ + f"{member}=True" for member, coord in self if coord is not None + ] + return f"{self.__class__.__name__}({', '.join(args)})" + + @staticmethod + def _filters( + members, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, + ): + """ + TDB: support coord_systems? + + """ + name = None + coord = None + + if isinstance(item, str): + name = item + else: + coord = item + + if name is not None: + members = {k: v for k, v in members.items() if v.name() == name} + + if standard_name is not None: + members = { + k: v + for k, v in members.items() + if v.standard_name == standard_name + } + + if long_name is not None: + members = { + k: v for k, v in members.items() if v.long_name == long_name + } + + if var_name is not None: + members = { + k: v for k, v in members.items() if v.var_name == var_name + } + + if axis is not None: + axis = axis.upper() + members = { + k: v for k, v in members.items() if guess_coord_axis(v) == axis + } + + if attributes is not None: + if not isinstance(attributes, Mapping): + emsg = ( + "The attributes keyword was expecting a dictionary " + f"type, but got a {type(attributes)} instead." + ) + raise ValueError(emsg) + + def _filter(coord): + return all( + k in coord.attributes + and _hexdigest(coord.attributes[k]) == _hexdigest(v) + for k, v in attributes.items() + ) + + members = {k: v for k, v in members.items() if _filter(v)} + + if coord is not None: + if hasattr(coord, "__class__") and coord.__class__ in ( + CoordMetadata, + DimCoordMetadata, + ): + target_metadata = coord + else: + target_metadata = coord.metadata + + members = { + k: v + for k, v in members.items() + if v.metadata == target_metadata + } + + return members + + def _remove(self, **kwargs): + result = {} + members = self.filters(**kwargs) + + for member in members.keys(): + if member in self.REQUIRED: + dmsg = f"Ignoring request to remove required coordinate {member!r}" + logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) + else: + result[member] = members[member] + setattr(self, member, None) + + return result + + def _setter(self, location, axis, coord, shape): + axis = axis.lower() + member = f"{location}_{axis}" + + # enforce the UGRID minimum coordinate requirement + if location == "node" and coord is None: + emsg = ( + f"{member!r} is a required coordinate, cannot set to 'None'." + ) + raise ValueError(emsg) + + if coord is not None: + if not isinstance(coord, AuxCoord): + emsg = f"{member!r} requires to be an 'AuxCoord', got {type(coord)}." + raise TypeError(emsg) + + guess_axis = guess_coord_axis(coord) + + if guess_axis and guess_axis.lower() != axis: + emsg = f"{member!r} requires a {axis}-axis like 'AuxCoord', got a {guess_axis.lower()}-axis like." + raise TypeError(emsg) + + if coord.climatological: + emsg = f"{member!r} cannot be a climatological 'AuxCoord'." + raise TypeError(emsg) + + if shape is not None and coord.shape != shape: + emsg = f"{member!r} requires to have shape {shape!r}, got {coord.shape!r}." + raise ValueError(emsg) + + self._members[member] = coord + + def _shape(self, location): + coord = getattr(self, f"{location}_x") + shape = coord.shape if coord is not None else None + if shape is None: + coord = getattr(self, f"{location}_y") + if coord is not None: + shape = coord.shape + return shape + + @property + def _edge_shape(self): + return self._shape(location="edge") + + @property + def _node_shape(self): + return self._shape(location="node") + + @property + def all_members(self): + return Mesh1DCoords(**self._members) + + @property + def edge_coords(self): + return MeshEdgeCoords(edge_x=self.edge_x, edge_y=self.edge_y) + + @property + def edge_x(self): + return self._members["edge_x"] + + @edge_x.setter + def edge_x(self, coord): + self._setter( + location="edge", axis="x", coord=coord, shape=self._edge_shape + ) + + @property + def edge_y(self): + return self._members["edge_y"] + + @edge_y.setter + def edge_y(self, coord): + self._setter( + location="edge", axis="y", coord=coord, shape=self._edge_shape + ) + + @property + def node_coords(self): + return MeshNodeCoords(node_x=self.node_x, node_y=self.node_y) + + @property + def node_x(self): + return self._members["node_x"] + + @node_x.setter + def node_x(self, coord): + self._setter( + location="node", axis="x", coord=coord, shape=self._node_shape + ) + + @property + def node_y(self): + return self._members["node_y"] + + @node_y.setter + def node_y(self, coord): + self._setter( + location="node", axis="y", coord=coord, shape=self._node_shape + ) + + def _add(self, coords): + member_x, member_y = coords._fields + + # deal with the special case where both members are changing + if coords[0] is not None and coords[1] is not None: + cache_x = self._members[member_x] + cache_y = self._members[member_y] + self._members[member_x] = None + self._members[member_y] = None + + try: + setattr(self, member_x, coords[0]) + setattr(self, member_y, coords[1]) + except (TypeError, ValueError): + # restore previous valid state + self._members[member_x] = cache_x + self._members[member_y] = cache_y + # now, re-raise the exception + raise + else: + # deal with the case where one or no member is changing + if coords[0] is not None: + setattr(self, member_x, coords[0]) + if coords[1] is not None: + setattr(self, member_y, coords[1]) + + def add(self, node_x=None, node_y=None, edge_x=None, edge_y=None): + """ + use self.remove(edge_x=True) to remove a coordinate e.g., using the + pattern self.add(edge_x=None) will not remove the edge_x coordinate + + """ + self._add(MeshNodeCoords(node_x, node_y)) + self._add(MeshEdgeCoords(edge_x, edge_y)) + + def filter(self, **kwargs): + result = self.filters(**kwargs) + + if len(result) > 1: + names = ", ".join( + f"{member}={coord!r}" for member, coord in result.items() + ) + emsg = ( + f"Expected to find exactly 1 coordinate, but found {len(result)}. " + f"They were: {names}." + ) + raise CoordinateNotFoundError(emsg) + + if len(result) == 0: + item = kwargs["item"] + if item is not None: + if not isinstance(item, str): + item = item.name() + name = ( + item + or kwargs["standard_name"] + or kwargs["long_name"] + or kwargs["var_name"] + or None + ) + name = "" if name is None else f"{name!r} " + emsg = ( + f"Expected to find exactly 1 {name}coordinate, but found none." + ) + raise CoordinateNotFoundError(emsg) + + return result + + def filters( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, + node=None, + edge=None, + face=None, + ): + # rationalise the tri-state behaviour + args = [node, edge, face] + state = not any(set(filter(lambda arg: arg is not None, args))) + node, edge, face = map( + lambda arg: arg if arg is not None else state, args + ) + + def func(args): + return args[1] is not None + + members = {} + if node: + members.update( + dict(filter(func, self.node_coords._asdict().items())) + ) + if edge: + members.update( + dict(filter(func, self.edge_coords._asdict().items())) + ) + if hasattr(self, "face_coords"): + if face: + members.update( + dict(filter(func, self.face_coords._asdict().items())) + ) + else: + dmsg = "Ignoring request to filter non-existent 'face_coords'" + logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) + + result = self._filters( + members, + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + axis=axis, + ) + + return result + + def remove( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, + node=None, + edge=None, + ): + return self._remove( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + axis=axis, + node=node, + edge=edge, + ) + + +class _Mesh2DCoordinateManager(_Mesh1DCoordinateManager): + OPTIONAL = ( + "edge_x", + "edge_y", + "face_x", + "face_y", + ) + + def __init__( + self, + node_x, + node_y, + edge_x=None, + edge_y=None, + face_x=None, + face_y=None, + ): + super().__init__(node_x, node_y, edge_x=edge_x, edge_y=edge_y) + + # optional coordinates + self.face_x = face_x + self.face_y = face_y + + @property + def _face_shape(self): + return self._shape(location="face") + + @property + def all_members(self): + return Mesh2DCoords(**self._members) + + @property + def face_coords(self): + return MeshFaceCoords(face_x=self.face_x, face_y=self.face_y) + + @property + def face_x(self): + return self._members["face_x"] + + @face_x.setter + def face_x(self, coord): + self._setter( + location="face", axis="x", coord=coord, shape=self._face_shape + ) + + @property + def face_y(self): + return self._members["face_y"] + + @face_y.setter + def face_y(self, coord): + self._setter( + location="face", axis="y", coord=coord, shape=self._face_shape + ) + + def add( + self, + node_x=None, + node_y=None, + edge_x=None, + edge_y=None, + face_x=None, + face_y=None, + ): + super().add(node_x=node_x, node_y=node_y, edge_x=edge_x, edge_y=edge_y) + self._add(MeshFaceCoords(face_x, face_y)) + + def remove( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, + node=None, + edge=None, + face=None, + ): + return self._remove( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + axis=axis, + node=node, + edge=edge, + face=face, + ) + + class _MeshConnectivityManagerMixin(ABC): REQUIRED = () OPTIONAL = () @@ -1065,371 +2006,25 @@ def face_node(self): return self._members["face_node_connectivity"] -# class Mesh(CFVariableMixin): -# """ -# -# .. todo:: -# -# .. questions:: -# -# - decide on the verbose/succinct version of __str__ vs __repr__ -# -# .. notes:: -# -# - the mesh is location agnostic -# -# - no need to support volume at mesh level, yet -# -# - topology_dimension -# - use for fast equality between Mesh instances -# - checking connectivity dimensionality, specifically the highest dimensonality of the -# "geometric element" being added i.e., reference the src_location/tgt_location -# - used to honour and enforce the minimum UGRID connectivity contract -# -# - support pickling -# -# - copy is off the table!! -# -# - MeshCoord.guess_points() -# - MeshCoord.to_AuxCoord() -# -# - don't provide public methods to return the coordinate and connectivity -# managers -# -# """ -# def __init__( -# self, -# topology_dimension, -# standard_name=None, -# long_name=None, -# var_name=None, -# units=None, -# attributes=None, -# node_dimension=None, -# edge_dimension=None, -# face_dimension=None, -# node_coords_and_axes=None, # [(coord, "x"), (coord, "y")] this is a stronger contract, not relying on guessing -# edge_coords_and_axes=None, # ditto -# face_coords_and_axes=None, # ditto -# connectivities=None, # [Connectivity, [Connectivity], ...] -# ): -# # TODO: support volumes. -# # TODO: support (coord, "z") -# -# # These are strings, if None is provided then assign the default string. -# self.node_dimension = node_dimension -# self.edge_dimension = edge_dimension -# self.face_dimension = face_dimension -# -# self._metadata_manager = metadata_manager_factory(MeshMetadata) -# -# self._metadata_manager.topology_dimension = topology_dimension -# -# self.standard_name = standard_name -# self.long_name = long_name -# self.var_name = var_name -# self.units = units -# self.attributes = attributes -# -# # based on the topology_dimension create the appropriate coordinate manager -# # with some intelligence -# self._coord_manager = ... -# -# # based on the topology_dimension create the appropriate connectivity manager -# # with some intelligence -# self._connectivity_manager = ... -# -# @property -# def all_coords(self): -# # return a namedtuple -# # coords = mesh.all_coords -# # coords.face_x, coords.edge_y -# pass -# -# @property -# def node_coords(self): -# # return a namedtuple -# # node_coords = mesh.node_coords -# # node_coords.x -# # node_coords.y -# pass -# -# @property -# def edge_coords(self): -# # as above -# pass -# -# @property -# def face_coords(self): -# # as above -# pass -# -# @property -# def all_connectivities(self): -# # return a namedtuple -# # conns = mesh.all_connectivities -# # conns.edge_node, conns.boundary_node -# pass -# -# @property -# def face_node_connectivity(self): -# # required -# return self._connectivity_manager.face_node -# -# @property -# def edge_node_connectivity(self): -# # optionally required -# return self._connectivity_manager.edge_node -# -# @property -# def face_edge_connectivity(self): -# # optional -# return self._connectivity_manager.face_edge -# -# @property -# def face_face_connectivity(self): -# # optional -# return self._connectivity_manager.face_face -# -# @property -# def edge_face_connectivity(self): -# # optional -# return self._connectivity_manager.edge_face -# -# @property -# def boundary_node_connectivity(self): -# # optional -# return self._connectivity_manager.boundard_node -# -# def coord(self, ...): -# # as Cube.coord i.e., ensure that one and only one coord-like is returned -# # otherwise raise and exception -# pass -# -# def coords( -# self, -# name_or_coord=None, -# standard_name=None, -# long_name=None, -# var_name=None, -# attributes=None, -# axis=None, -# node=False, -# edge=False, -# face=False, -# ): -# # do we support the coord_system kwargs? -# self._coord_manager.coords(...) -# -# def connectivity(self, ...): -# pass -# -# def connectivities( -# self, -# name_or_coord=None, -# standard_name=None, -# long_name=None, -# var_name=None, -# attributes=None, -# node=False, -# edge=False, -# face=False, -# ): -# pass -# -# def add_coords(self, node_x=None, node_y=None, edge_x=None, edge_y=None, face_x=None, face_y=None): -# # this supports adding a new coord to the manager, but also replacing an existing coord -# self._coord_manager.add(...) -# -# def add_connectivities(self, *args): -# # this supports adding a new connectivity to the manager, but also replacing an existing connectivity -# self._connectivity_manager.add(*args) -# -# def remove_coords(self, ...): -# # could provide the "name", "metadata", "coord"-instance -# # this could use mesh.coords() to find the coords -# self._coord_manager.remove(...) -# -# def remove_connectivities(self, ...): -# # needs to respect the minimum UGRID contract -# self._connectivity_manager.remove(...) -# -# def __eq__(self, other): -# # Full equality could be MASSIVE, so we want to avoid that. -# # Ideally we want a mesh signature from LFRic for comparison, although this would -# # limit Iris' relevance outside MO. -# # TL;DR: unknown quantity. -# raise NotImplemented -# -# def __ne__(self, other): -# # See __eq__ -# raise NotImplemented -# -# def __str__(self): -# pass -# -# def __repr__(self): -# pass -# -# def __unicode__(self, ...): -# pass -# -# def __getstate__(self): -# pass -# -# def __setstate__(self, state): -# pass -# -# def xml_element(self): -# pass -# -# # the MeshCoord will always have bounds, perhaps points. However the MeshCoord.guess_points() may -# # be a very useful part of its behaviour. -# # after using MeshCoord.guess_points(), the user may wish to add the associated MeshCoord.points into -# # the Mesh as face_coordinates. -# -# def to_AuxCoord(self, location, axis): -# # factory method -# # return the lazy AuxCoord(...) for the given location and axis -# -# def to_AuxCoords(self, location): -# # factory method -# # return the lazy AuxCoord(...), AuxCoord(...) -# -# def to_MeshCoord(self, location, axis): -# # factory method -# # return MeshCoord(..., location=location, axis=axis) -# # use Connectivity.indices_by_src() for fetching indices. -# -# def to_MeshCoords(self, location): -# # factory method -# # return MeshCoord(..., location=location, axis="x"), MeshCoord(..., location=location, axis="y") -# # use Connectivity.indices_by_src() for fetching indices. -# -# def dimension_names_reset(self, node=False, face=False, edge=False): -# # reset to defaults like this (suggestion) -# -# def dimension_names(self, node=None, face=None, edge=None): -# # e.g., only set self.node iff node != None. these attributes will -# # always be set to a user provided string or the default string. -# # return a namedtuple of dict-like -# -# @property -# def cf_role(self): -# return "mesh_topology" -# -# @property -# def topology_dimension(self): -# """ -# read-only -# -# """ -# return self._metadata_manager.topology_dimension -# -# # -# # - validate coord_systems -# # - validate climatological -# # - use guess_coord_axis (iris.utils) -# # - others? -# # -# class _Mesh1DCoordinateManager: -# REQUIRED = ( -# "node_x", -# "node_y", -# ) -# OPTIONAL = ( -# "edge_x", -# "edge_y", -# ) -# def __init__(self, node_x, node_y, edge_x=None, edge_y=None): -# # required -# self.node_x = node_x -# self.node_y = node_y -# # optional -# self.edge_x = edge_x -# self.edge_y = edge_y -# -# # WOO-GA - this can easily get out of sync with the self attributes. -# # choose the container wisely e.g., could be an dict..., also the self -# # attributes may need to be @property's that access the chosen _members container -# self._members = [ ... ] -# -# def __iter__(self): -# for member in self._members: -# yield member -# -# def __getstate__(self): -# pass -# -# def __setstate__(self, state): -# pass -# -# def coord(self, **kwargs): -# # see Cube.coord for pattern, checking for a single result -# return self.coords(**kwargs)[0] -# -# def coords(self, ...): -# # see Cube.coords for relevant patterns -# # return [ ... ] -# pass -# -# def add(self, **kwargs): -# pass -# -# def remove(self, ...): -# # needs to respect the minimum UGRID contract -# # use logging/warning to flag items not removed - highlight in doc-string -# # don't raise an exception -# -# def __str__(self): -# pass -# -# def __repr__(self): -# pass -# -# def __eq__(self, other): -# # Full equality could be MASSIVE, so we want to avoid that. -# # Ideally we want a mesh signature from LFRic for comparison, although this would -# # limit Iris' relevance outside MO. -# # TL;DR: unknown quantity. -# raise NotImplemented -# -# def __ne__(self, other): -# # See __eq__ -# raise NotImplemented -# -# -# class _Mesh2DCoordinateManager(_Mesh1DCoordinateManager): -# OPTIONAL = ( -# "edge_x", -# "edge_y", -# "face_x", -# "face_y", -# ) -# def __init__(self, node_x, node_y, edge_x=None, edge_y=None, face_x=None, face_y=None): -# # optional -# self.face_x = face_x -# self.face_y = face_y -# -# super().__init__(node_x, node_y, edge_x=edge_x, edge_y=edge_y) -# -# # does the order matter? -# self._members.extend([self.face_x, self.face_y]) -# -# - - #: Convenience collection of lenient metadata combine services. -SERVICES_COMBINE.append(ConnectivityMetadata.combine) -SERVICES.append(ConnectivityMetadata.combine) +_services = [ConnectivityMetadata.combine, MeshMetadata.combine] +SERVICES_COMBINE.extend(_services) +SERVICES.extend(_services) #: Convenience collection of lenient metadata difference services. -SERVICES_DIFFERENCE.append(ConnectivityMetadata.difference) -SERVICES.append(ConnectivityMetadata.difference) +_services = [ConnectivityMetadata.difference, MeshMetadata.difference] +SERVICES_DIFFERENCE.extend(_services) +SERVICES.extend(_services) #: Convenience collection of lenient metadata equality services. -SERVICES_EQUAL.extend( - [ConnectivityMetadata.__eq__, ConnectivityMetadata.equal] -) -SERVICES.extend([ConnectivityMetadata.__eq__, ConnectivityMetadata.equal]) +_services = [ + ConnectivityMetadata.__eq__, + ConnectivityMetadata.equal, + MeshMetadata.__eq__, + MeshMetadata.equal, +] +SERVICES_EQUAL.extend(_services) +SERVICES.extend(_services) + +del _services From 5cb558852eee9f60810a553ddec6217140a14c22 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 19 Feb 2021 14:59:53 +0000 Subject: [PATCH 06/22] Mesh dimension name abstraction. --- lib/iris/experimental/ugrid.py | 55 ++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 9722d2e7e9..7bf98f5b15 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -936,6 +936,33 @@ def __str__(self): args = [] return f"{self.__class__.__name__}({', '.join(args)})" + def _set_dimension_names(self, node, edge, face, reset=False): + inputs = (node, edge, face) + currents = ( + self.node_dimension, + self.edge_dimension, + self.face_dimension, + ) + zipped = zip(inputs, currents) + if reset: + node, edge, face = [ + None if check else current for check, current in zipped + ] + else: + node, edge, face = [new or current for new, current in zipped] + + self.node_dimension = node + self.edge_dimension = edge + self.face_dimension = face + + if self.topology_dimension == 1: + result = Mesh1DNames(self.node_dimension, self.edge_dimension) + else: + result = Mesh2DNames( + self.node_dimension, self.edge_dimension, self.face_dimension + ) + return result + @property def all_coords(self): return self._coord_manager.all_members @@ -1167,34 +1194,10 @@ def xml_element(self): # # use Connectivity.indices_by_src() for fetching indices. def dimension_names_reset(self, node=False, edge=False, face=False): - if node: - self.node_dimension = None - if edge: - self.edge_dimension = None - if face: - self.face_dimension = None - if self.topology_dimension == 1: - result = Mesh1DNames(self.node_dimension, self.edge_dimension) - else: - result = Mesh2DNames( - self.node_dimension, self.edge_dimension, self.face_dimension - ) - return result + self._set_dimension_names(node, edge, face, reset=True) def dimension_names(self, node=None, edge=None, face=None): - if node: - self.node_dimension = node - if edge: - self.edge_dimension = edge - if face: - self.face_dimension = face - if self.topology_dimension == 1: - result = Mesh1DNames(self.node_dimension, self.edge_dimension) - else: - result = Mesh2DNames( - self.node_dimension, self.edge_dimension, self.node_dimension - ) - return result + self._set_dimension_names(node, edge, face, reset=False) @property def cf_role(self): From bc8f42307aa1fca8fd65cfaa7160ef8ad3794661 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 19 Feb 2021 15:31:18 +0000 Subject: [PATCH 07/22] Align Cooord and Connectivity Managers filters methods. --- lib/iris/common/__init__.py | 5 +- lib/iris/experimental/ugrid.py | 151 +++++++++------------------------ 2 files changed, 43 insertions(+), 113 deletions(-) diff --git a/lib/iris/common/__init__.py b/lib/iris/common/__init__.py index 211c07a564..a4f71905b8 100644 --- a/lib/iris/common/__init__.py +++ b/lib/iris/common/__init__.py @@ -77,9 +77,8 @@ def attr_filter(instance_): result = [instance_ for instance_ in result if attr_filter(instance_)] if instance is not None: - if hasattr(instance, "__class__") and instance.__class__ in ( - CoordMetadata, - DimCoordMetadata, + if hasattr(instance, "__class__") and issubclass( + instance.__class__, BaseMetadata ): target_metadata = instance else: diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 7bf98f5b15..189fe18b0d 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -11,7 +11,7 @@ """ from abc import ABC, abstractmethod -from collections import Mapping, namedtuple +from collections import namedtuple from functools import wraps import dask.array as da @@ -20,10 +20,7 @@ from .. import _lazy_data as _lazy from ..common import filter_cf from ..common.metadata import ( - _hexdigest, BaseMetadata, - CoordMetadata, - DimCoordMetadata, metadata_manager_factory, SERVICES, SERVICES_COMBINE, @@ -1272,88 +1269,6 @@ def __str__(self): ] return f"{self.__class__.__name__}({', '.join(args)})" - @staticmethod - def _filters( - members, - item=None, - standard_name=None, - long_name=None, - var_name=None, - attributes=None, - axis=None, - ): - """ - TDB: support coord_systems? - - """ - name = None - coord = None - - if isinstance(item, str): - name = item - else: - coord = item - - if name is not None: - members = {k: v for k, v in members.items() if v.name() == name} - - if standard_name is not None: - members = { - k: v - for k, v in members.items() - if v.standard_name == standard_name - } - - if long_name is not None: - members = { - k: v for k, v in members.items() if v.long_name == long_name - } - - if var_name is not None: - members = { - k: v for k, v in members.items() if v.var_name == var_name - } - - if axis is not None: - axis = axis.upper() - members = { - k: v for k, v in members.items() if guess_coord_axis(v) == axis - } - - if attributes is not None: - if not isinstance(attributes, Mapping): - emsg = ( - "The attributes keyword was expecting a dictionary " - f"type, but got a {type(attributes)} instead." - ) - raise ValueError(emsg) - - def _filter(coord): - return all( - k in coord.attributes - and _hexdigest(coord.attributes[k]) == _hexdigest(v) - for k, v in attributes.items() - ) - - members = {k: v for k, v in members.items() if _filter(v)} - - if coord is not None: - if hasattr(coord, "__class__") and coord.__class__ in ( - CoordMetadata, - DimCoordMetadata, - ): - target_metadata = coord - else: - target_metadata = coord.metadata - - members = { - k: v - for k, v in members.items() - if v.metadata == target_metadata - } - - return members - def _remove(self, **kwargs): result = {} members = self.filters(**kwargs) @@ -1505,6 +1420,7 @@ def add(self, node_x=None, node_y=None, edge_x=None, edge_y=None): self._add(MeshEdgeCoords(edge_x, edge_y)) def filter(self, **kwargs): + # TODO: rationalise commonality with MeshConnectivityManager.filter and Cube.coord. result = self.filters(**kwargs) if len(result) > 1: @@ -1549,6 +1465,8 @@ def filters( edge=None, face=None, ): + # TBD: support coord_systems? + # rationalise the tri-state behaviour args = [node, edge, face] state = not any(set(filter(lambda arg: arg is not None, args))) @@ -1556,38 +1474,44 @@ def filters( lambda arg: arg if arg is not None else state, args ) - def func(args): - return args[1] is not None + def populated_coords(coords_tuple): + return list(filter(None, list(coords_tuple))) - members = {} + members = [] if node: - members.update( - dict(filter(func, self.node_coords._asdict().items())) - ) + members += populated_coords(self.node_coords) if edge: - members.update( - dict(filter(func, self.edge_coords._asdict().items())) - ) + members += populated_coords(self.edge_coords) if hasattr(self, "face_coords"): if face: - members.update( - dict(filter(func, self.face_coords._asdict().items())) - ) + members += populated_coords(self.face_coords) else: dmsg = "Ignoring request to filter non-existent 'face_coords'" logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) - result = self._filters( + if axis is not None: + axis = axis.upper() + members = [ + instance_ + for instance_ in members + if guess_coord_axis(instance_) == axis + ] + + result = filter_cf( members, item=item, standard_name=standard_name, long_name=long_name, var_name=var_name, attributes=attributes, - axis=axis, ) - return result + # Use the results to filter the _members dict for returning. + result_ids = [id(r) for r in result] + result_dict = { + k: v for k, v in self._members.items() if id(v) in result_ids + } + return result_dict def remove( self, @@ -1809,6 +1733,7 @@ def add(self, *connectivities): self._members = proposed_members def filter(self, **kwargs): + # TODO: rationalise commonality with MeshCoordManager.filter and Cube.coord. result = self.filters(**kwargs) if len(result) > 1: names = ", ".join( @@ -1849,14 +1774,7 @@ def filters( edge=None, face=None, ): - members = filter_cf( - [c for c in self._members.values() if c is not None], - item=item, - standard_name=standard_name, - long_name=long_name, - var_name=var_name, - attributes=attributes, - ) + members = [c for c in self._members.values() if c is not None] if cf_role is not None: members = [ @@ -1901,7 +1819,20 @@ def location_filter(instances_, parameter_, location_name_): ) logger.debug(message, extra=dict(cls=self.__class__.__name__)) - result_dict = {k: v for k, v in self._members.items() if v in members} + result = filter_cf( + members, + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + ) + + # Use the results to filter the _members dict for returning. + result_ids = [id(r) for r in result] + result_dict = { + k: v for k, v in self._members.items() if id(v) in result_ids + } return result_dict def remove( From 20fbeeddbf6360a7ba88d166c38570e0a053a1c4 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 19 Feb 2021 18:49:06 +0000 Subject: [PATCH 08/22] Completed Mesh class. --- lib/iris/common/__init__.py | 35 ++++ lib/iris/experimental/ugrid.py | 183 ++++++++++++------- lib/iris/tests/unit/common/test_filter_cf.py | 105 +++++++++++ 3 files changed, 257 insertions(+), 66 deletions(-) create mode 100644 lib/iris/tests/unit/common/test_filter_cf.py diff --git a/lib/iris/common/__init__.py b/lib/iris/common/__init__.py index a4f71905b8..47800b3ffc 100644 --- a/lib/iris/common/__init__.py +++ b/lib/iris/common/__init__.py @@ -24,6 +24,41 @@ def filter_cf( var_name=None, attributes=None, ): + """ + Filter a list of :class:`iris.common.CFVariableMixin` subclasses to fit + the given criteria. + + Kwargs: + + * item + Either + + (a) a :attr:`standard_name`, :attr:`long_name`, or + :attr:`var_name`. Defaults to value of `default` + (which itself defaults to `unknown`) as defined in + :class:`iris.common.CFVariableMixin`. + + (b) a 'coordinate' instance with metadata equal to that of + the desired coordinates. Accepts either a + :class:`iris.coords.DimCoord`, :class:`iris.coords.AuxCoord`, + :class:`iris.aux_factory.AuxCoordFactory`, + :class:`iris.common.CoordMetadata` or + :class:`iris.common.DimCoordMetadata` or + :class:`iris.experimental.ugrid.ConnectivityMetadata`. + * standard_name + The CF standard name of the desired coordinate. If None, does not + check for standard name. + * long_name + An unconstrained description of the coordinate. If None, does not + check for long_name. + * var_name + The netCDF variable name of the desired coordinate. If None, does + not check for var_name. + * attributes + A dictionary of attributes desired on the coordinates. If None, + does not check for attributes. + + """ name = None instance = None diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 189fe18b0d..570246d392 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -838,7 +838,7 @@ def __init__( attributes=None, edge_coords_and_axes=None, face_coords_and_axes=None, - # connectivities=None, + connectivities=None, node_dimension=None, edge_dimension=None, face_dimension=None, @@ -898,15 +898,18 @@ def normalise(location, axis): if self.topology_dimension == 1: self._coord_manager = _Mesh1DCoordinateManager(**kwargs) + self._connectivity_manager = _Mesh1DConnectivityManager( + *connectivities + ) elif self.topology_dimension == 2: self._coord_manager = _Mesh2DCoordinateManager(**kwargs) + self._connectivity_manager = _Mesh2DConnectivityManager( + *connectivities + ) else: emsg = f"Unsupported 'topology_dimension', got {topology_dimension!r}." raise NotImplementedError(emsg) - # based on the topology_dimension, create the appropriate connectivity manager - # self._connectivity_manager = ... - def __eq__(self, other): # TBD return NotImplemented @@ -1012,42 +1015,39 @@ def node_dimension(self, name): def node_coords(self): return self._coord_manager.node_coords - # @property - # def all_connectivities(self): - # # return a namedtuple - # # conns = mesh.all_connectivities - # # conns.edge_node, conns.boundary_node - # pass - # - # @property - # def face_node_connectivity(self): - # # required - # return self._connectivity_manager.face_node - # - # @property - # def edge_node_connectivity(self): - # # optionally required - # return self._connectivity_manager.edge_node - # - # @property - # def face_edge_connectivity(self): - # # optional - # return self._connectivity_manager.face_edge - # - # @property - # def face_face_connectivity(self): - # # optional - # return self._connectivity_manager.face_face - # - # @property - # def edge_face_connectivity(self): - # # optional - # return self._connectivity_manager.edge_face - # - # @property - # def boundary_node_connectivity(self): - # # optional - # return self._connectivity_manager.boundary_node + @property + def all_connectivities(self): + return self._connectivity_manager.all_members + + @property + def face_node_connectivity(self): + # required + return self._connectivity_manager.face_node + + @property + def edge_node_connectivity(self): + # optionally required + return self._connectivity_manager.edge_node + + @property + def face_edge_connectivity(self): + # optional + return self._connectivity_manager.face_edge + + @property + def face_face_connectivity(self): + # optional + return self._connectivity_manager.face_face + + @property + def edge_face_connectivity(self): + # optional + return self._connectivity_manager.edge_face + + @property + def boundary_node_connectivity(self): + # optional + return self._connectivity_manager.boundary_node def add_coords( self, @@ -1067,25 +1067,56 @@ def add_coords( face_y=face_y, ) - # def add_connectivities(self, *args): - # # this supports adding a new connectivity to the manager, but also replacing an existing connectivity - # self._connectivity_manager.add(*args) - - # def connectivities( - # self, - # name_or_coord=None, - # standard_name=None, - # long_name=None, - # var_name=None, - # attributes=None, - # node=False, - # edge=False, - # face=False, - # ): - # pass - - # def connectivity(self, ...): - # pass + def add_connectivities(self, *connectivities): + self._connectivity_manager.add(*connectivities) + + def connectivities( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + cf_role=None, + node=None, + edge=None, + face=None, + ): + return self._connectivity_manager.filters( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + cf_role=cf_role, + node=node, + edge=edge, + face=face, + ) + + def connectivity( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + cf_role=None, + node=None, + edge=None, + face=None, + ): + return self._connectivity_manager.filter( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + cf_role=cf_role, + node=node, + edge=edge, + face=face, + ) def coord( self, @@ -1135,9 +1166,29 @@ def coords( face=face, ) - # def remove_connectivities(self, ...): - # # needs to respect the minimum UGRID contract - # self._connectivity_manager.remove(...) + def remove_connectivities( + self, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + cf_role=None, + node=None, + edge=None, + face=None, + ): + return self._connectivity_manager.remove( + item=item, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + attributes=attributes, + cf_role=cf_role, + node=node, + edge=edge, + face=face, + ) def remove_coords( self, @@ -1151,7 +1202,7 @@ def remove_coords( edge=None, face=None, ): - self._coord_manager.remove( + return self._coord_manager.remove( item=item, standard_name=standard_name, long_name=long_name, @@ -1191,10 +1242,10 @@ def xml_element(self): # # use Connectivity.indices_by_src() for fetching indices. def dimension_names_reset(self, node=False, edge=False, face=False): - self._set_dimension_names(node, edge, face, reset=True) + return self._set_dimension_names(node, edge, face, reset=True) def dimension_names(self, node=None, edge=None, face=None): - self._set_dimension_names(node, edge, face, reset=False) + return self._set_dimension_names(node, edge, face, reset=False) @property def cf_role(self): @@ -1813,7 +1864,7 @@ def location_filter(instances_, parameter_, location_name_): # No need to actually modify filtering behaviour - already won't return # any face cf-roles if none are present. - if self.NDIM < 2: + if face and self.NDIM < 2: message = ( "Ignoring request to filter for non-existent 'face' cf-roles." ) diff --git a/lib/iris/tests/unit/common/test_filter_cf.py b/lib/iris/tests/unit/common/test_filter_cf.py new file mode 100644 index 0000000000..223def0175 --- /dev/null +++ b/lib/iris/tests/unit/common/test_filter_cf.py @@ -0,0 +1,105 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :func:`iris.common.filter_cf`. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import numpy as np + +from iris.common import filter_cf +from iris.common.metadata import CoordMetadata, DimCoordMetadata +from iris.coords import AuxCoord + +Mock = tests.mock.Mock + + +class Test_standard(tests.IrisTest): + def test_name(self): + name_one = Mock() + name_one.name.return_value = "one" + name_two = Mock() + name_two.name.return_value = "two" + input_list = [name_one, name_two] + result = filter_cf(input_list, item="one") + self.assertIn(name_one, result) + self.assertNotIn(name_two, result) + + def test_item(self): + coord = Mock(__class__=AuxCoord) + mock = Mock() + input_list = [coord, mock] + result = filter_cf(input_list, item=coord) + self.assertIn(coord, result) + self.assertNotIn(mock, result) + + def test_item_metadata(self): + coord = Mock(metadata=CoordMetadata) + dim_coord = Mock(metadata=DimCoordMetadata) + input_list = [coord, dim_coord] + result = filter_cf(input_list, item=coord) + self.assertIn(coord, result) + self.assertNotIn(dim_coord, result) + + def test_standard_name(self): + name_one = Mock(standard_name="one") + name_two = Mock(standard_name="two") + input_list = [name_one, name_two] + result = filter_cf(input_list, standard_name="one") + self.assertIn(name_one, result) + self.assertNotIn(name_two, result) + + def test_long_name(self): + name_one = Mock(long_name="one") + name_two = Mock(long_name="two") + input_list = [name_one, name_two] + result = filter_cf(input_list, long_name="one") + self.assertIn(name_one, result) + self.assertNotIn(name_two, result) + + def test_var_name(self): + name_one = Mock(var_name="one") + name_two = Mock(var_name="two") + input_list = [name_one, name_two] + result = filter_cf(input_list, var_name="one") + self.assertIn(name_one, result) + self.assertNotIn(name_two, result) + + def test_attributes(self): + # Confirm that this can handle attrib dicts including np arrays. + attrib_one_two = Mock( + attributes={"one": np.arange(1), "two": np.arange(2)} + ) + attrib_three_four = Mock( + attributes={"three": np.arange(3), "four": np.arange(4)} + ) + input_list = [attrib_one_two, attrib_three_four] + result = filter_cf(input_list, attributes=attrib_one_two.attributes) + self.assertIn(attrib_one_two, result) + self.assertNotIn(attrib_three_four, result) + + def test_invalid_attributes(self): + attrib_one = Mock(attributes={"one": 1}) + input_list = [attrib_one] + self.assertRaisesRegex( + ValueError, + ".*expecting a dictionary.*", + filter_cf, + input_list, + attributes="one", + ) + + def test_multiple_args(self): + coord_one = Mock(__class__=AuxCoord, long_name="one") + coord_two = Mock(__class__=AuxCoord, long_name="two") + input_list = [coord_one, coord_two] + result = filter_cf(input_list, item=coord_one, long_name="one") + self.assertIn(coord_one, result) + self.assertNotIn(coord_two, result) From 6675cd8a52c69f5b6f5cea55d93a86392c6ffd3d Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 11:46:13 +0000 Subject: [PATCH 09/22] filter_cf improvements. --- lib/iris/common/__init__.py | 84 ++++++++++++-------- lib/iris/cube.py | 10 +-- lib/iris/experimental/ugrid.py | 47 +++++------ lib/iris/tests/unit/common/test_filter_cf.py | 8 ++ 4 files changed, 80 insertions(+), 69 deletions(-) diff --git a/lib/iris/common/__init__.py b/lib/iris/common/__init__.py index 47800b3ffc..5f3e69c7b3 100644 --- a/lib/iris/common/__init__.py +++ b/lib/iris/common/__init__.py @@ -14,6 +14,7 @@ from .metadata import * from .mixin import * from .resolve import * +from ..util import guess_coord_axis def filter_cf( @@ -23,10 +24,17 @@ def filter_cf( long_name=None, var_name=None, attributes=None, + axis=None, ): """ - Filter a list of :class:`iris.common.CFVariableMixin` subclasses to fit - the given criteria. + Filter a collection of objects by their metadata to fit the given metadata + criteria. Criteria be one or both of: specific properties / other objects + carrying metadata to be matched. + + Args: + + * instances + An iterable of objects to be filtered. Kwargs: @@ -36,15 +44,15 @@ def filter_cf( (a) a :attr:`standard_name`, :attr:`long_name`, or :attr:`var_name`. Defaults to value of `default` (which itself defaults to `unknown`) as defined in - :class:`iris.common.CFVariableMixin`. + :class:`~iris.common.CFVariableMixin`. (b) a 'coordinate' instance with metadata equal to that of the desired coordinates. Accepts either a - :class:`iris.coords.DimCoord`, :class:`iris.coords.AuxCoord`, - :class:`iris.aux_factory.AuxCoordFactory`, - :class:`iris.common.CoordMetadata` or - :class:`iris.common.DimCoordMetadata` or - :class:`iris.experimental.ugrid.ConnectivityMetadata`. + :class:`~iris.coords.DimCoord`, :class:`~iris.coords.AuxCoord`, + :class:`~iris.aux_factory.AuxCoordFactory`, + :class:`~iris.common.CoordMetadata` or + :class:`~iris.common.DimCoordMetadata` or + :class:`~iris.experimental.ugrid.ConnectivityMetadata`. * standard_name The CF standard name of the desired coordinate. If None, does not check for standard name. @@ -57,40 +65,44 @@ def filter_cf( * attributes A dictionary of attributes desired on the coordinates. If None, does not check for attributes. + * axis + The desired coordinate axis, see + :func:`~iris.util.guess_coord_axis`. If None, does not check for + axis. Accepts the values 'X', 'Y', 'Z' and 'T' (case-insensitive). + + Returns: + A list of the objects supplied in the ``instances`` argument, limited + to only those that matched the given criteria. """ name = None - instance = None + obj = None if isinstance(item, str): name = item else: - instance = item + obj = item result = instances if name is not None: - result = [ - instance_ for instance_ in result if instance_.name() == name - ] + result = [instance for instance in result if instance.name() == name] if standard_name is not None: result = [ - instance_ - for instance_ in result - if instance_.standard_name == standard_name + instance + for instance in result + if instance.standard_name == standard_name ] if long_name is not None: result = [ - instance_ - for instance_ in result - if instance_.long_name == long_name + instance for instance in result if instance.long_name == long_name ] if var_name is not None: result = [ - instance_ for instance_ in result if instance_.var_name == var_name + instance for instance in result if instance.var_name == var_name ] if attributes is not None: @@ -101,28 +113,36 @@ def filter_cf( ) raise ValueError(msg) - def attr_filter(instance_): + def attr_filter(instance): return all( - k in instance_.attributes - and metadata._hexdigest(instance_.attributes[k]) + k in instance.attributes + and metadata._hexdigest(instance.attributes[k]) == metadata._hexdigest(v) for k, v in attributes.items() ) - result = [instance_ for instance_ in result if attr_filter(instance_)] + result = [instance for instance in result if attr_filter(instance)] + + if axis is not None: + axis = axis.upper() + result = [ + instance + for instance in result + if guess_coord_axis(instance) == axis + ] - if instance is not None: - if hasattr(instance, "__class__") and issubclass( - instance.__class__, BaseMetadata + if obj is not None: + if hasattr(obj, "__class__") and issubclass( + obj.__class__, BaseMetadata ): - target_metadata = instance + target_metadata = obj else: - target_metadata = instance.metadata + target_metadata = obj.metadata result = [ - instance_ - for instance_ in result - if instance_.metadata == target_metadata + instance + for instance in result + if instance.metadata == target_metadata ] return result diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 893d5a94fe..a5701f271b 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -1653,17 +1653,9 @@ def coords( long_name=long_name, var_name=var_name, attributes=attributes, + axis=axis, ) - if axis is not None: - axis = axis.upper() - guess_axis = iris.util.guess_coord_axis - coords_and_factories = [ - coord_ - for coord_ in coords_and_factories - if guess_axis(coord_) == axis - ] - if coord_system is not None: coords_and_factories = [ coord_ diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 570246d392..348a370db1 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -1540,14 +1540,6 @@ def populated_coords(coords_tuple): dmsg = "Ignoring request to filter non-existent 'face_coords'" logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) - if axis is not None: - axis = axis.upper() - members = [ - instance_ - for instance_ in members - if guess_coord_axis(instance_) == axis - ] - result = filter_cf( members, item=item, @@ -1555,6 +1547,7 @@ def populated_coords(coords_tuple): long_name=long_name, var_name=var_name, attributes=attributes, + axis=axis, ) # Use the results to filter the _members dict for returning. @@ -1829,38 +1822,36 @@ def filters( if cf_role is not None: members = [ - instance_ - for instance_ in members - if instance_.cf_role == cf_role + instance for instance in members if instance.cf_role == cf_role ] - def location_filter(instances_, parameter_, location_name_): - if parameter_ is False: - members_ = [ - instance_ - for instance_ in instances_ - if location_name_ - not in (instance_.src_location, instance_.tgt_location) + def location_filter(instances, loc_arg, loc_name): + if loc_arg is False: + filtered = [ + instance + for instance in instances + if loc_name + not in (instance.src_location, instance.tgt_location) ] - elif parameter_ is None: - members_ = instances_ + elif loc_arg is None: + filtered = instances else: # Interpret any other value as =True. - members_ = [ - instance_ - for instance_ in instances_ - if location_name_ - in (instance_.src_location, instance_.tgt_location) + filtered = [ + instance + for instance in instances + if loc_name + in (instance.src_location, instance.tgt_location) ] - return members_ + return filtered - for parameter, location_name in ( + for arg, loc in ( (node, "node"), (edge, "edge"), (face, "face"), ): - members = location_filter(members, parameter, location_name) + members = location_filter(members, arg, loc) # No need to actually modify filtering behaviour - already won't return # any face cf-roles if none are present. diff --git a/lib/iris/tests/unit/common/test_filter_cf.py b/lib/iris/tests/unit/common/test_filter_cf.py index 223def0175..03a3b35e5b 100644 --- a/lib/iris/tests/unit/common/test_filter_cf.py +++ b/lib/iris/tests/unit/common/test_filter_cf.py @@ -96,6 +96,14 @@ def test_invalid_attributes(self): attributes="one", ) + def test_axis(self): + axis_lon = Mock(standard_name="longitude") + axis_lat = Mock(standard_name="latitude") + input_list = [axis_lon, axis_lat] + result = filter_cf(input_list, axis="x") + self.assertIn(axis_lon, result) + self.assertNotIn(axis_lat, result) + def test_multiple_args(self): coord_one = Mock(__class__=AuxCoord, long_name="one") coord_two = Mock(__class__=AuxCoord, long_name="two") From 1d279c55394827d81b822ccfdeb14474e9b7749e Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 11:53:39 +0000 Subject: [PATCH 10/22] Moved filter_cf. --- lib/iris/common/__init__.py | 134 ------------------ lib/iris/common/metadata.py | 133 +++++++++++++++++ lib/iris/cube.py | 4 +- lib/iris/experimental/ugrid.py | 6 +- .../test_filter.py} | 25 ++-- 5 files changed, 150 insertions(+), 152 deletions(-) rename lib/iris/tests/unit/common/{test_filter_cf.py => metadata/test_filter.py} (83%) diff --git a/lib/iris/common/__init__.py b/lib/iris/common/__init__.py index 5f3e69c7b3..d8e8ba80ef 100644 --- a/lib/iris/common/__init__.py +++ b/lib/iris/common/__init__.py @@ -8,141 +8,7 @@ """ -from typing import Mapping - from .lenient import * from .metadata import * from .mixin import * from .resolve import * -from ..util import guess_coord_axis - - -def filter_cf( - instances, - item=None, - standard_name=None, - long_name=None, - var_name=None, - attributes=None, - axis=None, -): - """ - Filter a collection of objects by their metadata to fit the given metadata - criteria. Criteria be one or both of: specific properties / other objects - carrying metadata to be matched. - - Args: - - * instances - An iterable of objects to be filtered. - - Kwargs: - - * item - Either - - (a) a :attr:`standard_name`, :attr:`long_name`, or - :attr:`var_name`. Defaults to value of `default` - (which itself defaults to `unknown`) as defined in - :class:`~iris.common.CFVariableMixin`. - - (b) a 'coordinate' instance with metadata equal to that of - the desired coordinates. Accepts either a - :class:`~iris.coords.DimCoord`, :class:`~iris.coords.AuxCoord`, - :class:`~iris.aux_factory.AuxCoordFactory`, - :class:`~iris.common.CoordMetadata` or - :class:`~iris.common.DimCoordMetadata` or - :class:`~iris.experimental.ugrid.ConnectivityMetadata`. - * standard_name - The CF standard name of the desired coordinate. If None, does not - check for standard name. - * long_name - An unconstrained description of the coordinate. If None, does not - check for long_name. - * var_name - The netCDF variable name of the desired coordinate. If None, does - not check for var_name. - * attributes - A dictionary of attributes desired on the coordinates. If None, - does not check for attributes. - * axis - The desired coordinate axis, see - :func:`~iris.util.guess_coord_axis`. If None, does not check for - axis. Accepts the values 'X', 'Y', 'Z' and 'T' (case-insensitive). - - Returns: - A list of the objects supplied in the ``instances`` argument, limited - to only those that matched the given criteria. - - """ - name = None - obj = None - - if isinstance(item, str): - name = item - else: - obj = item - - result = instances - - if name is not None: - result = [instance for instance in result if instance.name() == name] - - if standard_name is not None: - result = [ - instance - for instance in result - if instance.standard_name == standard_name - ] - - if long_name is not None: - result = [ - instance for instance in result if instance.long_name == long_name - ] - - if var_name is not None: - result = [ - instance for instance in result if instance.var_name == var_name - ] - - if attributes is not None: - if not isinstance(attributes, Mapping): - msg = ( - "The attributes keyword was expecting a dictionary " - "type, but got a %s instead." % type(attributes) - ) - raise ValueError(msg) - - def attr_filter(instance): - return all( - k in instance.attributes - and metadata._hexdigest(instance.attributes[k]) - == metadata._hexdigest(v) - for k, v in attributes.items() - ) - - result = [instance for instance in result if attr_filter(instance)] - - if axis is not None: - axis = axis.upper() - result = [ - instance - for instance in result - if guess_coord_axis(instance) == axis - ] - - if obj is not None: - if hasattr(obj, "__class__") and issubclass( - obj.__class__, BaseMetadata - ): - target_metadata = obj - else: - target_metadata = obj.metadata - - result = [ - instance - for instance in result - if instance.metadata == target_metadata - ] - - return result diff --git a/lib/iris/common/metadata.py b/lib/iris/common/metadata.py index 40dccf9428..be7358ef2b 100644 --- a/lib/iris/common/metadata.py +++ b/lib/iris/common/metadata.py @@ -43,6 +43,9 @@ # https://www.unidata.ucar.edu/software/netcdf/docs/netcdf_data_set_components.html#object_name + +from ..util import guess_coord_axis + _TOKEN_PARSE = re.compile(r"""^[a-zA-Z0-9][\w\.\+\-@]*$""") # Configure the logger. @@ -1339,6 +1342,136 @@ def equal(self, other, lenient=None): return super().equal(other, lenient=lenient) +def filter( + instances, + item=None, + standard_name=None, + long_name=None, + var_name=None, + attributes=None, + axis=None, +): + """ + Filter a collection of objects by their metadata to fit the given metadata + criteria. Criteria be one or both of: specific properties / other objects + carrying metadata to be matched. + + Args: + + * instances + An iterable of objects to be filtered. + + Kwargs: + + * item + Either + + (a) a :attr:`standard_name`, :attr:`long_name`, or + :attr:`var_name`. Defaults to value of `default` + (which itself defaults to `unknown`) as defined in + :class:`~iris.common.CFVariableMixin`. + + (b) a 'coordinate' instance with metadata equal to that of + the desired coordinates. Accepts either a + :class:`~iris.coords.DimCoord`, :class:`~iris.coords.AuxCoord`, + :class:`~iris.aux_factory.AuxCoordFactory`, + :class:`~iris.common.CoordMetadata` or + :class:`~iris.common.DimCoordMetadata` or + :class:`~iris.experimental.ugrid.ConnectivityMetadata`. + * standard_name + The CF standard name of the desired coordinate. If None, does not + check for standard name. + * long_name + An unconstrained description of the coordinate. If None, does not + check for long_name. + * var_name + The netCDF variable name of the desired coordinate. If None, does + not check for var_name. + * attributes + A dictionary of attributes desired on the coordinates. If None, + does not check for attributes. + * axis + The desired coordinate axis, see + :func:`~iris.util.guess_coord_axis`. If None, does not check for + axis. Accepts the values 'X', 'Y', 'Z' and 'T' (case-insensitive). + + Returns: + A list of the objects supplied in the ``instances`` argument, limited + to only those that matched the given criteria. + + """ + name = None + obj = None + + if isinstance(item, str): + name = item + else: + obj = item + + result = instances + + if name is not None: + result = [instance for instance in result if instance.name() == name] + + if standard_name is not None: + result = [ + instance + for instance in result + if instance.standard_name == standard_name + ] + + if long_name is not None: + result = [ + instance for instance in result if instance.long_name == long_name + ] + + if var_name is not None: + result = [ + instance for instance in result if instance.var_name == var_name + ] + + if attributes is not None: + if not isinstance(attributes, Mapping): + msg = ( + "The attributes keyword was expecting a dictionary " + "type, but got a %s instead." % type(attributes) + ) + raise ValueError(msg) + + def attr_filter(instance): + return all( + k in instance.attributes + and _hexdigest(instance.attributes[k]) == _hexdigest(v) + for k, v in attributes.items() + ) + + result = [instance for instance in result if attr_filter(instance)] + + if axis is not None: + axis = axis.upper() + result = [ + instance + for instance in result + if guess_coord_axis(instance) == axis + ] + + if obj is not None: + if hasattr(obj, "__class__") and issubclass( + obj.__class__, BaseMetadata + ): + target_metadata = obj + else: + target_metadata = obj.metadata + + result = [ + instance + for instance in result + if instance.metadata == target_metadata + ] + + return result + + def metadata_manager_factory(cls, **kwargs): """ A class instance factory function responsible for manufacturing diff --git a/lib/iris/cube.py b/lib/iris/cube.py index a5701f271b..6b949e8fb7 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -41,8 +41,8 @@ CFVariableMixin, CubeMetadata, metadata_manager_factory, - filter_cf, ) +from iris.common.metadata import filter import iris.coord_systems import iris.coords import iris.exceptions @@ -1646,7 +1646,7 @@ def coords( coords_and_factories += list(self.aux_coords) coords_and_factories += list(self.aux_factories) - coords_and_factories = filter_cf( + coords_and_factories = filter( coords_and_factories, item=name_or_coord, standard_name=standard_name, diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 348a370db1..ec3fdee3dd 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -18,7 +18,6 @@ import numpy as np from .. import _lazy_data as _lazy -from ..common import filter_cf from ..common.metadata import ( BaseMetadata, metadata_manager_factory, @@ -26,6 +25,7 @@ SERVICES_COMBINE, SERVICES_EQUAL, SERVICES_DIFFERENCE, + filter, ) from ..common.lenient import _lenient_service as lenient_service from ..common.mixin import CFVariableMixin @@ -1540,7 +1540,7 @@ def populated_coords(coords_tuple): dmsg = "Ignoring request to filter non-existent 'face_coords'" logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) - result = filter_cf( + result = filter( members, item=item, standard_name=standard_name, @@ -1861,7 +1861,7 @@ def location_filter(instances, loc_arg, loc_name): ) logger.debug(message, extra=dict(cls=self.__class__.__name__)) - result = filter_cf( + result = filter( members, item=item, standard_name=standard_name, diff --git a/lib/iris/tests/unit/common/test_filter_cf.py b/lib/iris/tests/unit/common/metadata/test_filter.py similarity index 83% rename from lib/iris/tests/unit/common/test_filter_cf.py rename to lib/iris/tests/unit/common/metadata/test_filter.py index 03a3b35e5b..b17c2a9c2d 100644 --- a/lib/iris/tests/unit/common/test_filter_cf.py +++ b/lib/iris/tests/unit/common/metadata/test_filter.py @@ -4,7 +4,7 @@ # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. """ -Unit tests for the :func:`iris.common.filter_cf`. +Unit tests for the :func:`iris.common.filter`. """ @@ -14,8 +14,7 @@ import numpy as np -from iris.common import filter_cf -from iris.common.metadata import CoordMetadata, DimCoordMetadata +from iris.common.metadata import CoordMetadata, DimCoordMetadata, filter from iris.coords import AuxCoord Mock = tests.mock.Mock @@ -28,7 +27,7 @@ def test_name(self): name_two = Mock() name_two.name.return_value = "two" input_list = [name_one, name_two] - result = filter_cf(input_list, item="one") + result = filter(input_list, item="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -36,7 +35,7 @@ def test_item(self): coord = Mock(__class__=AuxCoord) mock = Mock() input_list = [coord, mock] - result = filter_cf(input_list, item=coord) + result = filter(input_list, item=coord) self.assertIn(coord, result) self.assertNotIn(mock, result) @@ -44,7 +43,7 @@ def test_item_metadata(self): coord = Mock(metadata=CoordMetadata) dim_coord = Mock(metadata=DimCoordMetadata) input_list = [coord, dim_coord] - result = filter_cf(input_list, item=coord) + result = filter(input_list, item=coord) self.assertIn(coord, result) self.assertNotIn(dim_coord, result) @@ -52,7 +51,7 @@ def test_standard_name(self): name_one = Mock(standard_name="one") name_two = Mock(standard_name="two") input_list = [name_one, name_two] - result = filter_cf(input_list, standard_name="one") + result = filter(input_list, standard_name="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -60,7 +59,7 @@ def test_long_name(self): name_one = Mock(long_name="one") name_two = Mock(long_name="two") input_list = [name_one, name_two] - result = filter_cf(input_list, long_name="one") + result = filter(input_list, long_name="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -68,7 +67,7 @@ def test_var_name(self): name_one = Mock(var_name="one") name_two = Mock(var_name="two") input_list = [name_one, name_two] - result = filter_cf(input_list, var_name="one") + result = filter(input_list, var_name="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -81,7 +80,7 @@ def test_attributes(self): attributes={"three": np.arange(3), "four": np.arange(4)} ) input_list = [attrib_one_two, attrib_three_four] - result = filter_cf(input_list, attributes=attrib_one_two.attributes) + result = filter(input_list, attributes=attrib_one_two.attributes) self.assertIn(attrib_one_two, result) self.assertNotIn(attrib_three_four, result) @@ -91,7 +90,7 @@ def test_invalid_attributes(self): self.assertRaisesRegex( ValueError, ".*expecting a dictionary.*", - filter_cf, + filter, input_list, attributes="one", ) @@ -100,7 +99,7 @@ def test_axis(self): axis_lon = Mock(standard_name="longitude") axis_lat = Mock(standard_name="latitude") input_list = [axis_lon, axis_lat] - result = filter_cf(input_list, axis="x") + result = filter(input_list, axis="x") self.assertIn(axis_lon, result) self.assertNotIn(axis_lat, result) @@ -108,6 +107,6 @@ def test_multiple_args(self): coord_one = Mock(__class__=AuxCoord, long_name="one") coord_two = Mock(__class__=AuxCoord, long_name="two") input_list = [coord_one, coord_two] - result = filter_cf(input_list, item=coord_one, long_name="one") + result = filter(input_list, item=coord_one, long_name="one") self.assertIn(coord_one, result) self.assertNotIn(coord_two, result) From 97cfb6da42e30fe184af38cdcc1be5636e80374b Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 11:55:09 +0000 Subject: [PATCH 11/22] Mesh connectivity manager namedtuples comment. --- lib/iris/experimental/ugrid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index ec3fdee3dd..36b9f7da6d 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -72,6 +72,7 @@ MeshEdgeCoords = namedtuple("MeshEdgeCoords", ["edge_x", "edge_y"]) MeshFaceCoords = namedtuple("MeshFaceCoords", ["face_x", "face_y"]) +# Mesh connectivity manager namedtuples. Mesh1DConnectivities = namedtuple("Mesh1DConnectivities", ["edge_node"]) Mesh2DConnectivities = namedtuple( "Mesh2DConnectivities", From a5648954d28c4d3a8c3353f268b304c79f52e97e Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 11:57:08 +0000 Subject: [PATCH 12/22] Mesh removed trailing underscores. --- lib/iris/experimental/ugrid.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 36b9f7da6d..b0f4a83212 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -975,10 +975,10 @@ def edge_dimension(self): @edge_dimension.setter def edge_dimension(self, name): if not name or not isinstance(name, str): - edge_dimension_ = f"Mesh{self.topology_dimension}d_edge" + edge_dimension = f"Mesh{self.topology_dimension}d_edge" else: - edge_dimension_ = name - self._metadata_manager.edge_dimension = edge_dimension_ + edge_dimension = name + self._metadata_manager.edge_dimension = edge_dimension @property def edge_coords(self): @@ -991,10 +991,10 @@ def face_dimension(self): @face_dimension.setter def face_dimension(self, name): if not name or not isinstance(name, str): - face_dimension_ = f"Mesh{self.topology_dimension}d_face" + face_dimension = f"Mesh{self.topology_dimension}d_face" else: - face_dimension_ = name - self._metadata_manager.face_dimension = face_dimension_ + face_dimension = name + self._metadata_manager.face_dimension = face_dimension @property def face_coords(self): @@ -1007,10 +1007,10 @@ def node_dimension(self): @node_dimension.setter def node_dimension(self, name): if not name or not isinstance(name, str): - node_dimension_ = f"Mesh{self.topology_dimension}d_node" + node_dimension = f"Mesh{self.topology_dimension}d_node" else: - node_dimension_ = name - self._metadata_manager.node_dimension = node_dimension_ + node_dimension = name + self._metadata_manager.node_dimension = node_dimension @property def node_coords(self): From 4a6ddb3a25b8f350a2b40530b6567d9196367141 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 12:02:11 +0000 Subject: [PATCH 13/22] Mesh _set_dimension_names improvements. --- lib/iris/experimental/ugrid.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index b0f4a83212..32540244ce 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -938,19 +938,19 @@ def __str__(self): return f"{self.__class__.__name__}({', '.join(args)})" def _set_dimension_names(self, node, edge, face, reset=False): - inputs = (node, edge, face) + args = (node, edge, face) currents = ( self.node_dimension, self.edge_dimension, self.face_dimension, ) - zipped = zip(inputs, currents) + zipped = zip(args, currents) if reset: node, edge, face = [ - None if check else current for check, current in zipped + None if arg else current for arg, current in zipped ] else: - node, edge, face = [new or current for new, current in zipped] + node, edge, face = [arg or current for arg, current in zipped] self.node_dimension = node self.edge_dimension = edge @@ -958,10 +958,16 @@ def _set_dimension_names(self, node, edge, face, reset=False): if self.topology_dimension == 1: result = Mesh1DNames(self.node_dimension, self.edge_dimension) - else: + elif self.topology_dimension == 2: result = Mesh2DNames( self.node_dimension, self.edge_dimension, self.face_dimension ) + else: + message = ( + f"Unsupported topology_dimension: {self.topology_dimension} ." + ) + raise NotImplementedError(message) + return result @property From 24c0a8731e6b8d9ae3c325313bc094adcb267184 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 12:03:19 +0000 Subject: [PATCH 14/22] Mesh import rationalisation. --- lib/iris/experimental/ugrid.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 32540244ce..e009d03f55 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -31,9 +31,8 @@ from ..common.mixin import CFVariableMixin from ..config import get_logger from ..coords import _DimensionalMetadata, AuxCoord -from ..exceptions import CoordinateNotFoundError +from ..exceptions import CoordinateNotFoundError, ConnectivityNotFoundError from ..util import guess_coord_axis -from .. import exceptions __all__ = [ @@ -1795,7 +1794,7 @@ def filter(self, **kwargs): f"Expected to find exactly 1 connectivity, but found " f"{len(result)}. They were: {names}." ) - raise exceptions.ConnectivityNotFoundError(message) + raise ConnectivityNotFoundError(message) elif len(result) == 0: item = kwargs["item"] _name = item @@ -1809,7 +1808,7 @@ def filter(self, **kwargs): f"Expected to find exactly 1 {bad_name} connectivity, " f"but found none." ) - raise exceptions.ConnectivityNotFoundError(message) + raise ConnectivityNotFoundError(message) return result From a39b26bb3e59c6d00c2d0153e0135a28644d98af Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 12:11:40 +0000 Subject: [PATCH 15/22] Mesh connectivity manager remove NDIM. --- lib/iris/experimental/ugrid.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index e009d03f55..2f1b2c9cf9 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -1681,14 +1681,15 @@ def remove( class _MeshConnectivityManagerMixin(ABC): REQUIRED = () OPTIONAL = () - NDIM = NotImplemented @abstractmethod def __init__(self, *connectivities): cf_roles = [c.cf_role for c in connectivities] for requisite in self.REQUIRED: if requisite not in cf_roles: - message = f"{self.NDIM}D meshes require a {requisite}." + message = ( + f"{self.__name__} requires a {requisite} Connectivity." + ) raise ValueError(message) self.ALL = self.REQUIRED + self.OPTIONAL @@ -1861,7 +1862,8 @@ def location_filter(instances, loc_arg, loc_name): # No need to actually modify filtering behaviour - already won't return # any face cf-roles if none are present. - if face and self.NDIM < 2: + supports_faces = any(["face" in role for role in self.ALL]) + if face and not supports_faces: message = ( "Ignoring request to filter for non-existent 'face' cf-roles." ) @@ -1924,7 +1926,6 @@ def remove( class _Mesh1DConnectivityManager(_MeshConnectivityManagerMixin): REQUIRED = ("edge_node_connectivity",) OPTIONAL = () - NDIM = 1 def __init__(self, *connectivities): super().__init__(*connectivities) @@ -1947,7 +1948,6 @@ class _Mesh2DConnectivityManager(_MeshConnectivityManagerMixin): "edge_face_connectivity", "boundary_node_connectivity", ) - NDIM = 2 def __init__(self, *connectivities): super().__init__(*connectivities) From 3ca9fa1e3cfaf5bb3709dcef1b23682c73235b2d Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 12:17:02 +0000 Subject: [PATCH 16/22] Connectivity manager use lazy indices_by_src(). --- lib/iris/experimental/ugrid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 2f1b2c9cf9..3e91fb9415 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -1240,12 +1240,12 @@ def xml_element(self): # def to_MeshCoord(self, location, axis): # # factory method # # return MeshCoord(..., location=location, axis=axis) - # # use Connectivity.indices_by_src() for fetching indices. + # # use Connectivity.indices_by_src() for fetching indices, passing in the lazy_indices() result as an argument. # # def to_MeshCoords(self, location): # # factory method # # return MeshCoord(..., location=location, axis="x"), MeshCoord(..., location=location, axis="y") - # # use Connectivity.indices_by_src() for fetching indices. + # # use Connectivity.indices_by_src for fetching indices, passing in the lazy_indices() result as an argument. def dimension_names_reset(self, node=False, edge=False, face=False): return self._set_dimension_names(node, edge, face, reset=True) @@ -1769,7 +1769,7 @@ def add(self, *connectivities): ) for location in locations: counts = [ - len(c.indices_by_src()) + len(c.indices_by_src(c.lazy_indices())) for c in proposed_members.values() if c is not None and c.src_location == location ] From 80fb8e0401865fb1d0aff1d898585545490f6f82 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 12:17:34 +0000 Subject: [PATCH 17/22] Connectivity manager clearer removal syntax. --- lib/iris/experimental/ugrid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 3e91fb9415..0d03cb874f 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -1909,11 +1909,11 @@ def remove( face=face, ) for cf_role in self.REQUIRED: - not_removed = removal_dict.pop(cf_role, None) - if not_removed: + excluded = removal_dict.pop(cf_role, None) + if excluded: message = ( f"Ignoring request to remove required connectivity " - f"({cf_role}: {not_removed!r})" + f"({cf_role}: {excluded!r})" ) logger.debug(message, extra=dict(cls=self.__class__.__name__)) From a8897abb48759742a48b84f8f82986c78bdb42b6 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 12:19:28 +0000 Subject: [PATCH 18/22] Connectivity manager don't override __init__. --- lib/iris/experimental/ugrid.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 0d03cb874f..957ccc7f42 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -1682,7 +1682,6 @@ class _MeshConnectivityManagerMixin(ABC): REQUIRED = () OPTIONAL = () - @abstractmethod def __init__(self, *connectivities): cf_roles = [c.cf_role for c in connectivities] for requisite in self.REQUIRED: @@ -1927,9 +1926,6 @@ class _Mesh1DConnectivityManager(_MeshConnectivityManagerMixin): REQUIRED = ("edge_node_connectivity",) OPTIONAL = () - def __init__(self, *connectivities): - super().__init__(*connectivities) - @property def all_members(self): return Mesh1DConnectivities(edge_node=self.edge_node) @@ -1949,9 +1945,6 @@ class _Mesh2DConnectivityManager(_MeshConnectivityManagerMixin): "boundary_node_connectivity", ) - def __init__(self, *connectivities): - super().__init__(*connectivities) - @property def all_members(self): return Mesh2DConnectivities( From f5be558813e7c30ab24ac903d99c62dc2817aa67 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 13:58:06 +0000 Subject: [PATCH 19/22] Connectivity manager correct base class syntax. --- lib/iris/experimental/ugrid.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 957ccc7f42..cf4235d076 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -1678,9 +1678,10 @@ def remove( ) -class _MeshConnectivityManagerMixin(ABC): - REQUIRED = () - OPTIONAL = () +class _MeshConnectivityManagerBase(ABC): + # Override these in subclasses. + REQUIRED: tuple = NotImplemented + OPTIONAL: tuple = NotImplemented def __init__(self, *connectivities): cf_roles = [c.cf_role for c in connectivities] @@ -1922,7 +1923,7 @@ def remove( return removal_dict -class _Mesh1DConnectivityManager(_MeshConnectivityManagerMixin): +class _Mesh1DConnectivityManager(_MeshConnectivityManagerBase): REQUIRED = ("edge_node_connectivity",) OPTIONAL = () @@ -1935,7 +1936,7 @@ def edge_node(self): return self._members["edge_node_connectivity"] -class _Mesh2DConnectivityManager(_MeshConnectivityManagerMixin): +class _Mesh2DConnectivityManager(_MeshConnectivityManagerBase): REQUIRED = ("face_node_connectivity",) OPTIONAL = ( "edge_node_connectivity", From d9177cc850ea03825a60824ff022dbb8deb2f5a7 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 15:05:41 +0000 Subject: [PATCH 20/22] Metadata filter hexdigest reference fix. --- lib/iris/common/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/common/metadata.py b/lib/iris/common/metadata.py index be7358ef2b..177caaf53c 100644 --- a/lib/iris/common/metadata.py +++ b/lib/iris/common/metadata.py @@ -1441,7 +1441,7 @@ def filter( def attr_filter(instance): return all( k in instance.attributes - and _hexdigest(instance.attributes[k]) == _hexdigest(v) + and hexdigest(instance.attributes[k]) == hexdigest(v) for k, v in attributes.items() ) From 2744cf1b6541fec440e649a1eaa1abd429f143a8 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 15:09:31 +0000 Subject: [PATCH 21/22] test_MeshMetadata fix. --- lib/iris/tests/unit/experimental/ugrid/test_MeshMetadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_MeshMetadata.py b/lib/iris/tests/unit/experimental/ugrid/test_MeshMetadata.py index cfc668fb88..105365c908 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_MeshMetadata.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_MeshMetadata.py @@ -398,10 +398,10 @@ def test_op_lenient_same_members_none(self): with mock.patch( "iris.common.metadata._LENIENT", return_value=True ): - self.assertTrue( + self.assertEqual( expected, lmetadata.combine(rmetadata)._asdict() ) - self.assertTrue( + self.assertEqual( expected, rmetadata.combine(lmetadata)._asdict() ) From 4cf57f06256a11355494a24040d056e2e050aa93 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 22 Feb 2021 15:36:30 +0000 Subject: [PATCH 22/22] Rename filter to metadata_filter. --- docs/src/userguide/cube_statistics.rst | 76 +++++++++---------- lib/iris/common/metadata.py | 2 +- lib/iris/cube.py | 4 +- lib/iris/experimental/ugrid.py | 6 +- ...test_filter.py => test_metadata_filter.py} | 30 +++++--- 5 files changed, 62 insertions(+), 56 deletions(-) rename lib/iris/tests/unit/common/metadata/{test_filter.py => test_metadata_filter.py} (80%) diff --git a/docs/src/userguide/cube_statistics.rst b/docs/src/userguide/cube_statistics.rst index 4eb016078e..d62a056f33 100644 --- a/docs/src/userguide/cube_statistics.rst +++ b/docs/src/userguide/cube_statistics.rst @@ -23,9 +23,9 @@ Collapsing Entire Data Dimensions In the :doc:`subsetting_a_cube` section we saw how to extract a subset of a cube in order to reduce either its dimensionality or its resolution. -Instead of simply extracting a sub-region of the data, -we can produce statistical functions of the data values -across a particular dimension, +Instead of simply extracting a sub-region of the data, +we can produce statistical functions of the data values +across a particular dimension, such as a 'mean over time' or 'minimum over latitude'. .. _cube-statistics_forecast_printout: @@ -57,9 +57,9 @@ For instance, suppose we have a cube: um_version: 7.3 -In this case we have a 4 dimensional cube; -to mean the vertical (z) dimension down to a single valued extent -we can pass the coordinate name and the aggregation definition to the +In this case we have a 4 dimensional cube; +to mean the vertical (z) dimension down to a single valued extent +we can pass the coordinate name and the aggregation definition to the :meth:`Cube.collapsed() ` method: >>> import iris.analysis @@ -88,8 +88,8 @@ we can pass the coordinate name and the aggregation definition to the mean: model_level_number -Similarly other analysis operators such as ``MAX``, ``MIN`` and ``STD_DEV`` -can be used instead of ``MEAN``, see :mod:`iris.analysis` for a full list +Similarly other analysis operators such as ``MAX``, ``MIN`` and ``STD_DEV`` +can be used instead of ``MEAN``, see :mod:`iris.analysis` for a full list of currently supported operators. For an example of using this functionality, the @@ -103,14 +103,14 @@ in the gallery takes a zonal mean of an ``XYT`` cube by using the Area Averaging ^^^^^^^^^^^^^^ -Some operators support additional keywords to the ``cube.collapsed`` method. -For example, :func:`iris.analysis.MEAN ` supports -a weights keyword which can be combined with +Some operators support additional keywords to the ``cube.collapsed`` method. +For example, :func:`iris.analysis.MEAN ` supports +a weights keyword which can be combined with :func:`iris.analysis.cartography.area_weights` to calculate an area average. -Let's use the same data as was loaded in the previous example. -Since ``grid_latitude`` and ``grid_longitude`` were both point coordinates -we must guess bound positions for them +Let's use the same data as was loaded in the previous example. +Since ``grid_latitude`` and ``grid_longitude`` were both point coordinates +we must guess bound positions for them in order to calculate the area of the grid boxes:: import iris.analysis.cartography @@ -155,24 +155,24 @@ including an example on taking a :ref:`global area-weighted mean Partially Reducing Data Dimensions ---------------------------------- -Instead of completely collapsing a dimension, other methods can be applied -to reduce or filter the number of data points of a particular dimension. +Instead of completely collapsing a dimension, other methods can be applied +to reduce or filter the number of data points of a particular dimension. Aggregation of Grouped Data ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The :meth:`Cube.aggregated_by ` operation -combines data for all points with the same value of a given coordinate. -To do this, you need a coordinate whose points take on only a limited set -of different values -- the *number* of these then determines the size of the +The :meth:`Cube.aggregated_by ` operation +combines data for all points with the same value of a given coordinate. +To do this, you need a coordinate whose points take on only a limited set +of different values -- the *number* of these then determines the size of the reduced dimension. -The :mod:`iris.coord_categorisation` module can be used to make such -'categorical' coordinates out of ordinary ones: The most common use is -to aggregate data over regular *time intervals*, +The :mod:`iris.coord_categorisation` module can be used to make such +'categorical' coordinates out of ordinary ones: The most common use is +to aggregate data over regular *time intervals*, such as by calendar month or day of the week. -For example, let's create two new coordinates on the cube +For example, let's create two new coordinates on the cube to represent the climatological seasons and the season year respectively:: import iris @@ -188,8 +188,8 @@ to represent the climatological seasons and the season year respectively:: .. note:: - The 'season year' is not the same as year number, because (e.g.) the months - Dec11, Jan12 + Feb12 all belong to 'DJF-12'. + The 'season year' is not the same as year number, because (e.g.) the months + Dec11, Jan12 + Feb12 all belong to 'DJF-12'. See :meth:`iris.coord_categorisation.add_season_year`. @@ -206,10 +206,10 @@ to represent the climatological seasons and the season year respectively:: iris.coord_categorisation.add_season_year(cube, 'time', name='season_year') annual_seasonal_mean = cube.aggregated_by( - ['clim_season', 'season_year'], + ['clim_season', 'season_year'], iris.analysis.MEAN) - + Printing this cube now shows that two extra coordinates exist on the cube: .. doctest:: aggregation @@ -238,20 +238,20 @@ These two coordinates can now be used to aggregate by season and climate-year: .. doctest:: aggregation >>> annual_seasonal_mean = cube.aggregated_by( - ... ['clim_season', 'season_year'], + ... ['clim_season', 'season_year'], ... iris.analysis.MEAN) >>> print(repr(annual_seasonal_mean)) - -The primary change in the cube is that the cube's data has been -reduced in the 'time' dimension by aggregation (taking means, in this case). -This has collected together all data points with the same values of season and + +The primary change in the cube is that the cube's data has been +reduced in the 'time' dimension by aggregation (taking means, in this case). +This has collected together all data points with the same values of season and season-year. The results are now indexed by the 19 different possible values of season and season-year in a new, reduced 'time' dimension. -We can see this by printing the first 10 values of season+year -from the original cube: These points are individual months, +We can see this by printing the first 10 values of season+year +from the original cube: These points are individual months, so adjacent ones are often in the same season: .. doctest:: aggregation @@ -271,7 +271,7 @@ so adjacent ones are often in the same season: djf 2007 djf 2007 -Compare this with the first 10 values of the new cube's coordinates: +Compare this with the first 10 values of the new cube's coordinates: All the points now have distinct season+year values: .. doctest:: aggregation @@ -294,7 +294,7 @@ All the points now have distinct season+year values: Because the original data started in April 2006 we have some incomplete seasons (e.g. there were only two months worth of data for 'mam-2006'). -In this case we can fix this by removing all of the resultant 'times' which +In this case we can fix this by removing all of the resultant 'times' which do not cover a three month period (note: judged here as > 3*28 days): .. doctest:: aggregation @@ -306,7 +306,7 @@ do not cover a three month period (note: judged here as > 3*28 days): >>> full_season_means -The final result now represents the seasonal mean temperature for 17 seasons +The final result now represents the seasonal mean temperature for 17 seasons from jja-2006 to jja-2010: .. doctest:: aggregation diff --git a/lib/iris/common/metadata.py b/lib/iris/common/metadata.py index 177caaf53c..e81c6b206c 100644 --- a/lib/iris/common/metadata.py +++ b/lib/iris/common/metadata.py @@ -1342,7 +1342,7 @@ def equal(self, other, lenient=None): return super().equal(other, lenient=lenient) -def filter( +def metadata_filter( instances, item=None, standard_name=None, diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 6b949e8fb7..e8b6d4a692 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -42,7 +42,7 @@ CubeMetadata, metadata_manager_factory, ) -from iris.common.metadata import filter +from iris.common.metadata import metadata_filter import iris.coord_systems import iris.coords import iris.exceptions @@ -1646,7 +1646,7 @@ def coords( coords_and_factories += list(self.aux_coords) coords_and_factories += list(self.aux_factories) - coords_and_factories = filter( + coords_and_factories = metadata_filter( coords_and_factories, item=name_or_coord, standard_name=standard_name, diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index cf4235d076..45c94dbf16 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -25,7 +25,7 @@ SERVICES_COMBINE, SERVICES_EQUAL, SERVICES_DIFFERENCE, - filter, + metadata_filter, ) from ..common.lenient import _lenient_service as lenient_service from ..common.mixin import CFVariableMixin @@ -1546,7 +1546,7 @@ def populated_coords(coords_tuple): dmsg = "Ignoring request to filter non-existent 'face_coords'" logger.debug(dmsg, extra=dict(cls=self.__class__.__name__)) - result = filter( + result = metadata_filter( members, item=item, standard_name=standard_name, @@ -1869,7 +1869,7 @@ def location_filter(instances, loc_arg, loc_name): ) logger.debug(message, extra=dict(cls=self.__class__.__name__)) - result = filter( + result = metadata_filter( members, item=item, standard_name=standard_name, diff --git a/lib/iris/tests/unit/common/metadata/test_filter.py b/lib/iris/tests/unit/common/metadata/test_metadata_filter.py similarity index 80% rename from lib/iris/tests/unit/common/metadata/test_filter.py rename to lib/iris/tests/unit/common/metadata/test_metadata_filter.py index b17c2a9c2d..dafb50554b 100644 --- a/lib/iris/tests/unit/common/metadata/test_filter.py +++ b/lib/iris/tests/unit/common/metadata/test_metadata_filter.py @@ -4,7 +4,7 @@ # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. """ -Unit tests for the :func:`iris.common.filter`. +Unit tests for the :func:`iris.common.metadata_filter`. """ @@ -14,7 +14,11 @@ import numpy as np -from iris.common.metadata import CoordMetadata, DimCoordMetadata, filter +from iris.common.metadata import ( + CoordMetadata, + DimCoordMetadata, + metadata_filter, +) from iris.coords import AuxCoord Mock = tests.mock.Mock @@ -27,7 +31,7 @@ def test_name(self): name_two = Mock() name_two.name.return_value = "two" input_list = [name_one, name_two] - result = filter(input_list, item="one") + result = metadata_filter(input_list, item="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -35,7 +39,7 @@ def test_item(self): coord = Mock(__class__=AuxCoord) mock = Mock() input_list = [coord, mock] - result = filter(input_list, item=coord) + result = metadata_filter(input_list, item=coord) self.assertIn(coord, result) self.assertNotIn(mock, result) @@ -43,7 +47,7 @@ def test_item_metadata(self): coord = Mock(metadata=CoordMetadata) dim_coord = Mock(metadata=DimCoordMetadata) input_list = [coord, dim_coord] - result = filter(input_list, item=coord) + result = metadata_filter(input_list, item=coord) self.assertIn(coord, result) self.assertNotIn(dim_coord, result) @@ -51,7 +55,7 @@ def test_standard_name(self): name_one = Mock(standard_name="one") name_two = Mock(standard_name="two") input_list = [name_one, name_two] - result = filter(input_list, standard_name="one") + result = metadata_filter(input_list, standard_name="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -59,7 +63,7 @@ def test_long_name(self): name_one = Mock(long_name="one") name_two = Mock(long_name="two") input_list = [name_one, name_two] - result = filter(input_list, long_name="one") + result = metadata_filter(input_list, long_name="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -67,7 +71,7 @@ def test_var_name(self): name_one = Mock(var_name="one") name_two = Mock(var_name="two") input_list = [name_one, name_two] - result = filter(input_list, var_name="one") + result = metadata_filter(input_list, var_name="one") self.assertIn(name_one, result) self.assertNotIn(name_two, result) @@ -80,7 +84,9 @@ def test_attributes(self): attributes={"three": np.arange(3), "four": np.arange(4)} ) input_list = [attrib_one_two, attrib_three_four] - result = filter(input_list, attributes=attrib_one_two.attributes) + result = metadata_filter( + input_list, attributes=attrib_one_two.attributes + ) self.assertIn(attrib_one_two, result) self.assertNotIn(attrib_three_four, result) @@ -90,7 +96,7 @@ def test_invalid_attributes(self): self.assertRaisesRegex( ValueError, ".*expecting a dictionary.*", - filter, + metadata_filter, input_list, attributes="one", ) @@ -99,7 +105,7 @@ def test_axis(self): axis_lon = Mock(standard_name="longitude") axis_lat = Mock(standard_name="latitude") input_list = [axis_lon, axis_lat] - result = filter(input_list, axis="x") + result = metadata_filter(input_list, axis="x") self.assertIn(axis_lon, result) self.assertNotIn(axis_lat, result) @@ -107,6 +113,6 @@ def test_multiple_args(self): coord_one = Mock(__class__=AuxCoord, long_name="one") coord_two = Mock(__class__=AuxCoord, long_name="two") input_list = [coord_one, coord_two] - result = filter(input_list, item=coord_one, long_name="one") + result = metadata_filter(input_list, item=coord_one, long_name="one") self.assertIn(coord_one, result) self.assertNotIn(coord_two, result)