docs: Generating sphinx datastructure tables on the fly (#3344)

* Generating sphinx datastructure tables on the fly * Updating doc build instructions * add cmake dependency of geosx_docs on geosx_build_datastructure_tables * Using absolute paths for datastructure tables --------- Co-authored-by: Randolph Settgast <[email protected]>
GEOS-DEV · Sep 13, 2024 · dc604e5 · dc604e5
1 parent 67b6cd5
commit dc604e5
Show file tree

Hide file tree

Showing 492 changed files with 161 additions and 10,664 deletions.
diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,7 @@
 /Debug/
 uberenv_libs
 spack-*.txt
+/src/docs/sphinx/datastructure
 *~
 __pycache__
 

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -15,6 +15,8 @@ build:
     nodejs: "16"
     python: "3.10"
   jobs:
+    pre_build:
+    - python3 scripts/SchemaToRSTDocumentation.py -s src/coreComponents/schema/schema.xsd -o src/docs/sphinx
     post_install:
     - npm install -g [email protected]
     - npm install -g @mermaid-js/[email protected]

diff --git a/scripts/SchemaToRSTDocumentation.py b/scripts/SchemaToRSTDocumentation.py
@@ -1,7 +1,7 @@
 import os
 import re
-import sys
-#import numpy as np
+import shutil
+import argparse
 from xml.etree import ElementTree as etree
 
 
@@ -15,7 +15,6 @@ def comment(self, data):
 
 def writeTableRST(file_name, values):
     L = [[len(x) for x in row] for row in values]
-    #  M = tuple(np.amax(np.array(L), axis=0))
 
     # np isn't in the docker images for our CI
     MAX = [None] * len(L[0])
@@ -211,107 +210,97 @@ def buildTableValues(type_map, link_string='XML', include_defaults=True):
     return table_values
 
 
-# Config
-if len(sys.argv) < 2:
-    print("Usage: %s <schema_dir>" % sys.argv[0])
-    sys.exit(1)
-schema_dir = sys.argv[1]
-schema_name = os.path.join(schema_dir, 'schema.xsd')
-additional_documentation_name = os.path.join(schema_dir, 'schema.xsd.other')
-complete_output = os.path.join(schema_dir, '../../docs/sphinx/CompleteXMLSchema')
-output_folder = os.path.join(schema_dir, 'docs')
-sphinx_path = '../../coreComponents/schema/docs'
-xsd = '{http://www.w3.org/2001/XMLSchema}'
-
-# Parse the input/non-input schemas
-parser = etree.XMLParser(target=TreeBuilderWithComments())
-include_tree = etree.parse(schema_name, parser=parser)
-include_root = include_tree.getroot()
-input_attribute_map = buildAttributeMap(include_root)
-
-parser = etree.XMLParser(target=TreeBuilderWithComments())
-include_tree = etree.parse(additional_documentation_name, parser=parser)
-include_root = include_tree.getroot()
-other_attribute_map = buildAttributeMap(include_root)
-
-# Check for non-unique (ignoring case) links
-input_keys = sorted(input_attribute_map.keys())
-input_keys_lower = [k.lower() for k in input_keys]
-input_keys_count = [input_keys_lower.count(k) for k in input_keys_lower]
-input_repeated_keys = [input_keys[ii] for ii in range(0, len(input_keys)) if input_keys_count[ii] > 1]
-
-other_keys = sorted(other_attribute_map.keys())
-other_keys_lower = [k.lower() for k in other_keys]
-other_keys_count = [other_keys_lower.count(k) for k in other_keys_lower]
-other_repeated_keys = [other_keys[ii] for ii in range(0, len(other_keys)) if other_keys_count[ii] > 1]
-
-if ((len(input_repeated_keys) > 0) | (len(other_repeated_keys) > 0)):
-    print('Duplicate input documentation table names:')
-    print(input_repeated_keys)
-    print('Duplicate other documentation table names:')
-    print(other_repeated_keys)
-    raise ValueError('Duplicate data structure names are not allowed due to .rst limitations (case-insensitive)!')
-
-# Setup directory
-os.system('mkdir -p %s' % (output_folder))
-
-# Keep track of existing/touched files
-old_files = [f for f in os.listdir(output_folder) if '.rst' in f]
-touched_files = []
-
-# Build documentation tables
-with open('%s.rst' % (complete_output), 'w') as output_handle:
-    # Write the file header
-    output_handle.write('###################\n')
-    output_handle.write('Datastructure Index\n')
-    output_handle.write('###################\n\n')
-
-    # Parse the input schema definitions
-    output_handle.write('************************\n')
-    output_handle.write('Input Schema Definitions\n')
-    output_handle.write('************************\n\n')
-
-    output_handle.write(':download:`XML Schema <%s/../schema.xsd>`\n\n' % (sphinx_path))
-
-    for type_name in sorted(input_attribute_map.keys()):
-        # Write the individual tables
-        table_values = buildTableValues(input_attribute_map[type_name])
-        writeTableRST('%s/%s.rst' % (output_folder, type_name), table_values)
-        touched_files.append('%s.rst' % (type_name))
-
-        # Write to the master list
-        element_header = 'Element: %s' % (type_name)
-        output_handle.write('\n.. _XML_%s:\n\n' % (type_name))
-        output_handle.write('%s\n' % (element_header))
-        output_handle.write('=' * len(element_header) + '\n')
-        output_handle.write('.. include:: %s/%s.rst\n\n' % (sphinx_path, type_name))
-
-    # Parse the non-input schema definitions
-    output_handle.write('*************************\n')
-    output_handle.write('Datastructure Definitions\n')
-    output_handle.write('*************************\n\n')
-
-    for type_name in sorted(other_attribute_map.keys()):
-        # Write the individual tables
-        table_values = buildTableValues(other_attribute_map[type_name],
-                                        link_string='DATASTRUCTURE',
-                                        include_defaults=False)
-        writeTableRST('%s/%s_other.rst' % (output_folder, type_name), table_values)
-        touched_files.append('%s_other.rst' % (type_name))
-
-        # Write to the master list
-        element_header = 'Datastructure: %s' % (type_name)
-        output_handle.write('\n.. _DATASTRUCTURE_%s:\n\n' % (type_name))
-        output_handle.write('%s\n' % (element_header))
-        output_handle.write('=' * len(element_header) + '\n')
-        output_handle.write('.. include:: %s/%s_other.rst\n\n' % (sphinx_path, type_name))
-
-# Check for any untouched tables
-untouched_files = []
-for f in old_files:
-    if f not in touched_files:
-        untouched_files.append(f)
-
-if len(untouched_files):
-    tmp = '\n  '.join(untouched_files)
-    raise ValueError('Obsolete autogenerated .rst table files detected:%s' % (tmp))
+def main(schema_name='schema.xsd', output_folder='./', xsd='{http://www.w3.org/2001/XMLSchema}'):
+    """
+    Build RST Documentation Tables
+    """
+    # Setup folders
+    additional_documentation_name = f'{schema_name}.other'
+    output_folder = os.path.abspath(os.path.expanduser(output_folder))
+    datastructure_folder = os.path.join(output_folder, 'datastructure')
+    summary_file = os.path.join(datastructure_folder, 'CompleteXMLSchema.rst')
+    os.makedirs(datastructure_folder, exist_ok=True)
+    shutil.copy(schema_name, os.path.join(datastructure_folder, 'schema.xsd'))
+    shutil.copy(additional_documentation_name, os.path.join(datastructure_folder, 'schema.xsd.other'))
+
+    # Parse the input/non-input schemas
+    parser = etree.XMLParser(target=TreeBuilderWithComments())
+    include_tree = etree.parse(schema_name, parser=parser)
+    include_root = include_tree.getroot()
+    input_attribute_map = buildAttributeMap(include_root)
+
+    parser = etree.XMLParser(target=TreeBuilderWithComments())
+    include_tree = etree.parse(additional_documentation_name, parser=parser)
+    include_root = include_tree.getroot()
+    other_attribute_map = buildAttributeMap(include_root)
+
+    # Check for non-unique (ignoring case) links
+    input_keys = sorted(input_attribute_map.keys())
+    input_keys_lower = [k.lower() for k in input_keys]
+    input_keys_count = [input_keys_lower.count(k) for k in input_keys_lower]
+    input_repeated_keys = [input_keys[ii] for ii in range(0, len(input_keys)) if input_keys_count[ii] > 1]
+
+    other_keys = sorted(other_attribute_map.keys())
+    other_keys_lower = [k.lower() for k in other_keys]
+    other_keys_count = [other_keys_lower.count(k) for k in other_keys_lower]
+    other_repeated_keys = [other_keys[ii] for ii in range(0, len(other_keys)) if other_keys_count[ii] > 1]
+
+    if ((len(input_repeated_keys) > 0) | (len(other_repeated_keys) > 0)):
+        print('Duplicate input documentation table names:')
+        print(input_repeated_keys)
+        print('Duplicate other documentation table names:')
+        print(other_repeated_keys)
+        raise ValueError('Duplicate data structure names are not allowed due to .rst limitations (case-insensitive)!')
+
+    # Build documentation tables
+    with open(summary_file, 'w') as output_handle:
+        # Write the file header
+        output_handle.write('###################\n')
+        output_handle.write('Datastructure Index\n')
+        output_handle.write('###################\n\n')
+
+        # Parse the input schema definitions
+        output_handle.write('************************\n')
+        output_handle.write('Input Schema Definitions\n')
+        output_handle.write('************************\n\n')
+
+        output_handle.write(':download:`XML Schema <schema.xsd>`\n\n')
+
+        for type_name in sorted(input_attribute_map.keys()):
+            # Write the individual tables
+            table_values = buildTableValues(input_attribute_map[type_name])
+            writeTableRST('%s/%s.rst' % (datastructure_folder, type_name), table_values)
+
+            # Write to the master list
+            element_header = 'Element: %s' % (type_name)
+            output_handle.write('\n.. _XML_%s:\n\n' % (type_name))
+            output_handle.write('%s\n' % (element_header))
+            output_handle.write('=' * len(element_header) + '\n')
+            output_handle.write('.. include:: %s.rst\n\n' % (type_name))
+
+        # Parse the non-input schema definitions
+        output_handle.write('*************************\n')
+        output_handle.write('Datastructure Definitions\n')
+        output_handle.write('*************************\n\n')
+
+        for type_name in sorted(other_attribute_map.keys()):
+            # Write the individual tables
+            table_values = buildTableValues(other_attribute_map[type_name],
+                                            link_string='DATASTRUCTURE',
+                                            include_defaults=False)
+            writeTableRST('%s/%s_other.rst' % (datastructure_folder, type_name), table_values)
+
+            # Write to the master list
+            element_header = 'Datastructure: %s' % (type_name)
+            output_handle.write('\n.. _DATASTRUCTURE_%s:\n\n' % (type_name))
+            output_handle.write('%s\n' % (element_header))
+            output_handle.write('=' * len(element_header) + '\n')
+            output_handle.write('.. include:: %s_other.rst\n\n' % (type_name))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-s', '--schema', type=str, help='GEOS schema file', default='schema.xsd')
+    parser.add_argument('-o', '--output', type=str, help='Output directory', default='./')
+    args = parser.parse_args()
+    main(schema_name=args.schema, output_folder=args.output)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -169,9 +169,9 @@ if( ENABLE_XML_UPDATES AND ENABLE_MPI AND UNIX AND NOT CMAKE_HOST_APPLE AND NOT
                      COMMENT "Generating XML schema"
                      )
 
-  add_custom_target( geosx_update_rst_tables
+  add_custom_target( geosx_build_datastructure_tables
                      ALL
-                     COMMAND python3 ${SCRIPT_DIR}/SchemaToRSTDocumentation.py ${SCHEMA_DIR} >update_rst_tables.log 2>&1 || (cat update_rst_tables.log && exit 1)
+                     COMMAND python3 ${SCRIPT_DIR}/SchemaToRSTDocumentation.py -s ${SCHEMA_DIR}/schema.xsd -o ${CMAKE_SOURCE_DIR}/docs/sphinx >update_rst_tables.log 2>&1 || (cat update_rst_tables.log && exit 1)
                      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
                      DEPENDS geosx_generate_schema
                      COMMENT "Generating schema-dependent RST files"
@@ -181,7 +181,7 @@ if( ENABLE_XML_UPDATES AND ENABLE_MPI AND UNIX AND NOT CMAKE_HOST_APPLE AND NOT
                      ALL
                      COMMAND bash ${SCRIPT_DIR}/validateXMLFiles.bash -g ${SCHEMA_DIR}/schema.xsd ${CMAKE_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/../examples ${CMAKE_SOURCE_DIR}/../inputFiles
                      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-                     DEPENDS geosx_update_rst_tables
+                     DEPENDS geosx_generate_schema
                      COMMENT "Validating all XML files in the repository against the schema"
                      )
 endif()
@@ -192,6 +192,7 @@ endif()
 
 if( SPHINX_FOUND )
   blt_add_sphinx_target( geosx_docs )
+  add_dependencies( geosx_docs geosx_build_datastructure_tables )
 endif()
 
 

diff --git a/src/coreComponents/constitutive/docs/BiotPorosity.rst b/src/coreComponents/constitutive/docs/BiotPorosity.rst
@@ -33,7 +33,7 @@ block.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/BiotPorosity.rst
+.. include:: /docs/sphinx/datastructure/BiotPorosity.rst
 
 Example
 =======================

diff --git a/src/coreComponents/constitutive/docs/BlackOilFluid.rst b/src/coreComponents/constitutive/docs/BlackOilFluid.rst
@@ -83,7 +83,7 @@ In order to use the model, GEOS must be built with ``-DENABLE_PVTPACKAGE=ON`` (d
 
 The following attributes are supported:
 
-.. include:: ../../../coreComponents/schema/docs/BlackOilFluid.rst
+.. include:: /docs/sphinx/datastructure/BlackOilFluid.rst
 
 Supported phase names are:
 

diff --git a/src/coreComponents/constitutive/docs/BrooksCoreyCapillaryPressure.rst b/src/coreComponents/constitutive/docs/BrooksCoreyCapillaryPressure.rst
@@ -47,7 +47,7 @@ node.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/BrooksCoreyCapillaryPressure.rst
+.. include:: /docs/sphinx/datastructure/BrooksCoreyCapillaryPressure.rst
 
 Below are some comments on the model parameters:
 

diff --git a/src/coreComponents/constitutive/docs/BrooksCoreyRelativePermeability.rst b/src/coreComponents/constitutive/docs/BrooksCoreyRelativePermeability.rst
@@ -35,7 +35,7 @@ node.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/BrooksCoreyRelativePermeability.rst
+.. include:: /docs/sphinx/datastructure/BrooksCoreyRelativePermeability.rst
 
 Below are some comments on the model parameters.
 

diff --git a/src/coreComponents/constitutive/docs/CO2BrineFluid.rst b/src/coreComponents/constitutive/docs/CO2BrineFluid.rst
@@ -244,7 +244,7 @@ The models are represented by ``<CO2BrinePhillipsFluid>``, ``<CO2BrineEzrokhiFlu
 
 The following attributes are supported:
 
-.. include:: ../../../coreComponents/schema/docs/CO2BrinePhillipsFluid.rst
+.. include:: /docs/sphinx/datastructure/CO2BrinePhillipsFluid.rst
 
 Supported phase names are:
 

diff --git a/src/coreComponents/constitutive/docs/CompositionalMultiphaseFluid.rst b/src/coreComponents/constitutive/docs/CompositionalMultiphaseFluid.rst
@@ -199,7 +199,7 @@ In order to use the model, GEOS must be built with ``-DENABLE_PVTPACKAGE=ON`` (d
 
 The following attributes are supported:
 
-.. include:: ../../../coreComponents/schema/docs/CompositionalMultiphaseFluid.rst
+.. include:: /docs/sphinx/datastructure/CompositionalMultiphaseFluid.rst
 
 Supported phase names are:
 

diff --git a/src/coreComponents/constitutive/docs/CompressibleSinglePhaseFluid.rst b/src/coreComponents/constitutive/docs/CompressibleSinglePhaseFluid.rst
@@ -36,7 +36,7 @@ The model is represented by ``<CompressibleSinglePhaseFluid>`` node in the input
 
 The following attributes are supported:
 
-.. include:: ../../../coreComponents/schema/docs/CompressibleSinglePhaseFluid.rst
+.. include:: /docs/sphinx/datastructure/CompressibleSinglePhaseFluid.rst
 
 Example
 =========================

diff --git a/src/coreComponents/constitutive/docs/ConstantPermeability.rst b/src/coreComponents/constitutive/docs/ConstantPermeability.rst
@@ -17,7 +17,7 @@ Parameters
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/ConstantPermeability.rst
+.. include:: /docs/sphinx/datastructure/ConstantPermeability.rst
 
 
 Example

diff --git a/src/coreComponents/constitutive/docs/ExponentialDecayPermeability.rst b/src/coreComponents/constitutive/docs/ExponentialDecayPermeability.rst
@@ -30,7 +30,7 @@ block.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/ExponentialDecayPermeability.rst
+.. include:: /docs/sphinx/datastructure/ExponentialDecayPermeability.rst
 
 
 Example

diff --git a/src/coreComponents/constitutive/docs/KozenyCarmanPermeability.rst b/src/coreComponents/constitutive/docs/KozenyCarmanPermeability.rst
@@ -32,7 +32,7 @@ block.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/CarmanKozenyPermeability.rst
+.. include:: /docs/sphinx/datastructure/CarmanKozenyPermeability.rst
 
 
 

diff --git a/src/coreComponents/constitutive/docs/ParallelPlatesPermeability.rst b/src/coreComponents/constitutive/docs/ParallelPlatesPermeability.rst
@@ -35,7 +35,7 @@ block.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/ParallelPlatesPermeability.rst
+.. include:: /docs/sphinx/datastructure/ParallelPlatesPermeability.rst
 
 
 .. _Witherspoon et al.: https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/WR016i006p01016
diff --git a/src/coreComponents/constitutive/docs/PressurePorosity.rst b/src/coreComponents/constitutive/docs/PressurePorosity.rst
@@ -23,7 +23,7 @@ Parameters
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/PressurePorosity.rst
+.. include:: /docs/sphinx/datastructure/PressurePorosity.rst
 
 Example
 =======================

diff --git a/src/coreComponents/constitutive/docs/SlipDependentPermeability.rst b/src/coreComponents/constitutive/docs/SlipDependentPermeability.rst
@@ -30,7 +30,7 @@ block.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/SlipDependentPermeability.rst
+.. include:: /docs/sphinx/datastructure/SlipDependentPermeability.rst
 
 
 Example

diff --git a/src/coreComponents/constitutive/docs/TableCapillaryPressure.rst b/src/coreComponents/constitutive/docs/TableCapillaryPressure.rst
@@ -46,7 +46,7 @@ node.
 
 The following attributes are supported:
 
-.. include:: /coreComponents/schema/docs/TableCapillaryPressure.rst
+.. include:: /docs/sphinx/datastructure/TableCapillaryPressure.rst
 
 Below are some comments on the model parameters.
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,7 +17,7 @@ Parameters @@
     The following attributes are supported:
-    .. include:: /coreComponents/schema/docs/ConstantPermeability.rst
+    .. include:: /docs/sphinx/datastructure/ConstantPermeability.rst
     Example
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
Expand Up		@@ -32,7 +32,7 @@ block.

		The following attributes are supported:

		.. include:: /coreComponents/schema/docs/CarmanKozenyPermeability.rst
		.. include:: /docs/sphinx/datastructure/CarmanKozenyPermeability.rst



Expand Down