API reference: detect missing class pages & populate all gaps

docs/source/check_docs_api_coverage.py

@@ -1,4 +1,6 @@
-"""Check the method-coverage of all classes in docs/source/class.rst.
+"""Check the class-coverage and method-coverage of the API reference.
+
+All classes are extracted and checked for an entry in docs/source/class.rst
 
 All non-private methods of all such classes are checked for having an
 entry in their corresponding class's file in docs/source/class/
@@ -12,6 +14,8 @@
 
 """
 
+import inspect
+import re
 import os
 import sys
 
@@ -29,9 +33,13 @@
 if not source.endswith("source"):
     raise ValueError(f"Given directory {source} does not end with 'source'")
 
+n_undocumented_classes = 0
 n_undocumented_methods = 0
 n_missing_files = 0
 
+duplicate_method_entries = []
+
+
 for core in ("", "_core"):
     if core:
         if package.__name__ != "cfdm":
@@ -43,41 +51,96 @@
     with open(os.path.join(source, "class" + core + ".rst")) as f:
         api_contents = f.read()
 
+    # TODO: after #958 is resolved, replace this by grabbing all classes from
+    # '__all__', which defines the public API and therefore what must be
+    # include - and do the same for the methods etc.
     class_names = [
-        i.split(".")[-1]
-        for i in api_contents.split("\n")
-        if package.__name__ + "." in i
+        name
+        for name, klass in inspect.getmembers(package, inspect.isclass)
+        if klass.__module__.startswith(package.__name__ + ".")
+        # Because of docstring substitution in cfdm, all functions imported
+        # from there emerge as classes, with:
+        # type= <class 'cfdm.core.meta.docstringrewrite.DocstringRewriteMeta'>
+        # so when we try to extract classes only we end up with a lot of
+        # functions mixed in. To filter these out, we can use the fact that
+        # the functions emerge from just some modules, notably .functions etc.:
+        and not klass.__module__.startswith(package.__name__ + ".functions")
+        and not klass.__module__.startswith(package.__name__ + ".constants")
+        # This just counts top-level read-write i.e. cf.read and cf.write
+        and not klass.__module__.startswith(package.__name__ + ".read_write")
     ]
 
     for class_name in class_names:
         class_name = class_name.rstrip()
 
+        full_class_name = f"{package.__name__}.{class_name}"
+
+        if full_class_name not in api_contents:
+            print(f"Class {full_class_name} not in docs/source/class{core}.rst")
+            n_missing_files += 1
+            n_undocumented_classes += 1
+            continue
+
         klass = getattr(package, class_name)
 
         methods = [
             method for method in dir(klass) if not method.startswith("_")
         ]
-        class_name = ".".join([package.__name__, class_name])
 
-        rst_file = os.path.join(source, "class", class_name + ".rst")
+        rst_file = os.path.join(source, "class", full_class_name + ".rst")
 
         try:
             with open(rst_file) as f:
                 rst_contents = f.read()
 
             for method in methods:
-                method = ".".join([class_name, method])
-                if method not in rst_contents:
+                method = ".".join([full_class_name, method])
+                count = rst_contents.count(method)
+                if count == 0:
                     n_undocumented_methods += 1
-                    print(f"Method {method} not in {rst_file}")
+                    print(
+                        f"Method {method} not in "
+                        f"{os.path.join(source, 'class', rst_file)}"
+                    )
+                elif count > 1:
+                    # The method appears more than once, but may be a
+                    # sub-string of another method name, e.g. this gets caught:
+                    #     [cfdm.List.]nc_set_variable
+                    # due to the presence of this method:
+                    #     [cfdm.List.]nc_set_variable_groups
+                    # so we must account for that. Checking next character
+                    # of duplicate(s) is something other than a newline or
+                    # whitespace, seems robust and simplest.
+                    end_loc = [
+                        m.end(0) for m in re.finditer(method, rst_contents)
+                    ]
+                    chars = [rst_contents[c] for c in [e for e in end_loc]]
+
+                    # Any character that isn't a newline or whitespace
+                    # indicates another method which the method is a substring
+                    # of and can be excluded. If there are still duplicates,
+                    # we have genuine duplicate listing entries to report.
+                    if chars.count("\n") + chars.count(" ") > 1:
+                        duplicate_method_entries.append(method)
+
         except FileNotFoundError:
             n_missing_files += 1
-            print(f"File {rst_file} does not exist")
+            print(f"File {rst_file} does not exist for existing class ")
 
 if n_undocumented_methods or n_missing_files:
     raise ValueError(
-        f"Found {n_undocumented_methods} undocumented methods and "
+        f"Found {n_undocumented_classes} undocumented classes, "
+        f"{n_undocumented_methods} undocumented methods and "
         f"{n_missing_files} missing .rst files"
     )
 
+if duplicate_method_entries:
+    duplicate_method_entries.sort()
+    entries = "\n".join(duplicate_method_entries)  # can't set \n in f-string!
+    print(
+        "WARNING: some methods are listed multiple times inside one class "
+        "file/page. Decide if the duplicates are intended and if not remove "
+        f"them. They are:\n{entries}\n"
+    )
+
 print("All methods are documented")

docs/source/class.rst

@@ -32,7 +32,16 @@ Domain class
    :toctree: class/
 
    cf.Domain
-
+
+Domain list class
+-----------------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: class/
+
+   cf.DomainList
+
 Metadata component classes
 --------------------------
 
@@ -51,15 +60,16 @@ Metadata component classes
    cf.DomainTopology
    cf.FieldAncillary
 
-Constructs class
-----------------
+Constructs classes
+------------------
 
 .. autosummary::
    :nosignatures:
    :toctree: class/
 
    cf.Constructs
-
+   cf.ConstructList
+
 Coordinate component classes
 ----------------------------
 
@@ -70,18 +80,22 @@ Coordinate component classes
    cf.Bounds
    cf.CoordinateConversion
    cf.Datum
-
+   cf.InteriorRing
+
 Data classes
 ------------
 
 .. autosummary::
    :nosignatures:
    :toctree: class/
 
+   cf.AggregatedArray
    cf.Data
    cf.H5netcdfArray
    cf.NetCDF4Array
    cf.FullArray
+   cf.ScipyNetcdfFileArray
+   cf.PyfiveArray
    cf.UMArray
    cf.ZarrArray
 
@@ -102,6 +116,7 @@ Classes that support the creation and storage of compressed arrays.
    cf.RaggedIndexedArray
    cf.RaggedIndexedContiguousArray
    cf.SubsampledArray
+   cf.TiePointIndex
    cf.Quantization
 
 Data UGRID classes
@@ -116,7 +131,18 @@ Classes that support the creation and storage of UGRID-related arrays.
    cf.BoundsFromNodesArray
    cf.CellConnectivityArray
    cf.PointTopologyArray
-
+   cf.NodeCountProperties
+   cf.PartNodeCountProperties
+
+CF Data Model Implementation class
+----------------------------------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: class/
+
+   cf.CFImplementation
+
 Miscellaneous classes
 ---------------------
 
@@ -131,4 +157,13 @@ Miscellaneous classes
    cf.RegridOperator
    cf.Constant
    cf.Configuration
-
+   cf.InterpolationParameter
+
+Deprecated classes
+------------------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: class/
+
+   cf.NetCDFArray

docs/source/class/cf.AggregatedArray.rst

@@ -0,0 +1,58 @@
+.. currentmodule:: cf
+.. default-role:: obj
+
+.. _aggregatedarray:
+
+cf.AggregatedArray
+==================
+
+----
+
+.. autoclass:: cf.AggregatedArray
+   :no-members:
+   :no-inherited-members:
+
+Methods
+-------
+
+.. rubric:: Methods
+
+.. autosummary::
+   :nosignatures:
+   :toctree: ../method/
+   :template: method.rst
+
+   ~cf.AggregatedArray.Units
+   ~cf.AggregatedArray.array
+   ~cf.AggregatedArray.astype
+   ~cf.AggregatedArray.close
+   ~cf.AggregatedArray.copy
+   ~cf.AggregatedArray.dtype
+   ~cf.AggregatedArray.file_directory
+   ~cf.AggregatedArray.get_address
+   ~cf.AggregatedArray.get_attributes
+   ~cf.AggregatedArray.get_calendar
+   ~cf.AggregatedArray.get_compression_type
+   ~cf.AggregatedArray.get_filename
+   ~cf.AggregatedArray.get_fragment_array
+   ~cf.AggregatedArray.get_fragment_array_shape
+   ~cf.AggregatedArray.get_fragment_type
+   ~cf.AggregatedArray.get_fragmented_dimensions
+   ~cf.AggregatedArray.get_mask
+   ~cf.AggregatedArray.get_missing_values
+   ~cf.AggregatedArray.get_storage_options
+   ~cf.AggregatedArray.get_storage_protocol
+   ~cf.AggregatedArray.get_units
+   ~cf.AggregatedArray.get_unpack
+   ~cf.AggregatedArray.get_variable
+   ~cf.AggregatedArray.has_remote_storage_protocol
+   ~cf.AggregatedArray.ndim
+   ~cf.AggregatedArray.open
+   ~cf.AggregatedArray.replace_directory
+   ~cf.AggregatedArray.replace_filename
+   ~cf.AggregatedArray.shape
+   ~cf.AggregatedArray.size
+   ~cf.AggregatedArray.subarray_shapes
+   ~cf.AggregatedArray.subarrays
+   ~cf.AggregatedArray.to_dask_array
+   ~cf.AggregatedArray.to_memory