Merge branch 'master' into logical. Also testgres.pubsub was added to the Sphinx templates and module header was added as well.

Author: zilder

LICENSE

@@ -1,6 +1,6 @@
 testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses.
 
-Copyright (c) 2016-2017, Postgres Professional
+Copyright (c) 2016-2018, Postgres Professional
 
 Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies.
 

docs/source/testgres.rst

@@ -17,21 +17,14 @@ testgres.backup
     :undoc-members:
     :show-inheritance:
 
-testgres.cache
---------------
-
-.. automodule:: testgres.cache
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 testgres.config
 ---------------
 
 .. automodule:: testgres.config
     :members:
     :undoc-members:
     :show-inheritance:
+    :member-order: groupwise
 
 testgres.connection
 -------------------
@@ -41,30 +34,6 @@ testgres.connection
     :undoc-members:
     :show-inheritance:
 
-testgres.consts
----------------
-
-.. automodule:: testgres.consts
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-testgres.decorators
--------------------
-
-.. automodule:: testgres.decorators
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-testgres.defaults
------------------
-
-.. automodule:: testgres.defaults
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 testgres.enums
 --------------
 
@@ -81,35 +50,28 @@ testgres.exceptions
     :undoc-members:
     :show-inheritance:
 
-testgres.logger
----------------
-
-.. automodule:: testgres.logger
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 testgres.node
 -------------
 
-.. automodule:: testgres.node
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-testgres.utils
---------------
+.. autoclass:: testgres.node.PostgresNode
+   :members:
 
-.. automodule:: testgres.utils
-    :members:
-    :undoc-members:
-    :show-inheritance:
+   .. automethod:: __init__
 
+.. autoclass:: testgres.node.ProcessProxy
+   :members:
 
-Module contents
+testgres.pubsub
 ---------------
 
-.. automodule:: testgres
-    :members:
-    :undoc-members:
-    :show-inheritance:
+.. automodule:: testgres.pubsub
+
+.. autoclass:: testgres.node.Publication
+   :members:
+
+   .. automethod:: __init__
+
+.. autoclass:: testgres.node.Subscription
+   :members:
+
+   .. automethod:: __init__

testgres/api.py

@@ -28,17 +28,15 @@
 ...             replica.catchup()  # wait until changes are visible
 ...             print(replica.execute('postgres', 'select count(*) from test'))
 PostgresNode(name='...', port=..., base_dir='...')
-[(3,)]
-
-Copyright (c) 2016, Postgres Professional
+[(3L,)]
 """
-
-from functools import wraps
-
 from .node import PostgresNode
 
 
-@wraps(PostgresNode.__init__)
 def get_new_node(name=None, base_dir=None, **kwargs):
+    """
+    Simply a wrapper around :class:`.PostgresNode` constructor.
+    See :meth:`.PostgresNode.__init__` for details.
+    """
     # NOTE: leave explicit 'name' and 'base_dir' for compatibility
     return PostgresNode(name=name, base_dir=base_dir, **kwargs)

testgres/backup.py

@@ -43,7 +43,7 @@ def __init__(self,
         Create a new backup.
 
         Args:
-            node: PostgresNode we're going to backup.
+            node: :class:`.PostgresNode` we're going to backup.
             base_dir: where should we store it?
             username: database user name.
             xlog_method: none | fetch | stream (see docs)
@@ -74,15 +74,14 @@ def __init__(self,
 
         data_dir = os.path.join(self.base_dir, DATA_DIR)
 
-        # yapf: disable
         _params = [
             get_bin_path("pg_basebackup"),
             "-p", str(node.port),
             "-h", node.host,
             "-U", username,
             "-D", data_dir,
             "-X", xlog_method.value
-        ]
+        ]  # yapf: disable
         execute_utility(_params, self.log_file)
 
     def __enter__(self):
@@ -137,16 +136,15 @@ def spawn_primary(self, name=None, destroy=True):
             destroy: should we convert this backup into a node?
 
         Returns:
-            New instance of PostgresNode.
+            New instance of :class:`.PostgresNode`.
         """
 
         # Prepare a data directory for this node
         base_dir = self._prepare_dir(destroy)
 
         # Build a new PostgresNode
         from .node import PostgresNode
-        with clean_on_error(PostgresNode(name=name,
-                                         base_dir=base_dir)) as node:
+        with clean_on_error(PostgresNode(name=name, base_dir=base_dir)) as node:
 
             # New nodes should always remove dir tree
             node._should_rm_dirs = True
@@ -166,7 +164,7 @@ def spawn_replica(self, name=None, destroy=True, slot=None):
             destroy: should we convert this backup into a node?
 
         Returns:
-            New instance of PostgresNode.
+            New instance of :class:`.PostgresNode`.
         """
 
         # Build a new PostgresNode
@@ -180,6 +178,11 @@ def spawn_replica(self, name=None, destroy=True, slot=None):
             return node
 
     def cleanup(self):
+        """
+        Remove all files that belong to this backup.
+        No-op if it's been converted to a PostgresNode (destroy=True).
+        """
+
         if self._available:
             self._available = False
             rmtree(self.base_dir, ignore_errors=True)

testgres/config.py

@@ -13,42 +13,40 @@
 
 class GlobalConfig(object):
     """
-    Global config (override default settings).
-
-    Attributes:
-        cache_initdb:           shall we use cached initdb instance?
-        cached_initdb_dir:      shall we create a temp dir for cached initdb?
-        cached_initdb_unique:   shall we assign new node a unique system id?
-
-        cache_pg_config:        shall we cache pg_config results?
-
-        temp_dir:               base temp dir for nodes with default 'base_dir'.
-
-        use_python_logging:     use python logging configuration for all nodes.
-        error_log_lines:        N of log lines to be shown in exception (0=inf).
-
-        node_cleanup_full:          shall we remove EVERYTHING (including logs)?
-        node_cleanup_on_good_exit:  remove base_dir on nominal __exit__().
-        node_cleanup_on_bad_exit:   remove base_dir on __exit__() via exception.
-
-    NOTE: attributes must not be callable or begin with __.
+    Global configuration object which allows user to override default settings.
     """
+    # NOTE: attributes must not be callable or begin with __.
 
     cache_initdb = True
-    _cached_initdb_dir = None
+    """ shall we use cached initdb instance? """
+
     cached_initdb_unique = False
+    """ shall we give new node a unique system id? """
 
     cache_pg_config = True
+    """ shall we cache pg_config results? """
 
     use_python_logging = False
+    """ enable python logging subsystem (see logger.py). """
+
     error_log_lines = 20
+    """ N of log lines to be shown in exceptions (0=inf). """
 
     node_cleanup_full = True
+    """ shall we remove EVERYTHING (including logs)? """
+
     node_cleanup_on_good_exit = True
+    """ remove base_dir on nominal __exit__(). """
+
     node_cleanup_on_bad_exit = False
+    """ remove base_dir on __exit__() via exception. """
+
+    _cached_initdb_dir = None
+    """ underlying class attribute for cached_initdb_dir property """
 
     @property
     def cached_initdb_dir(self):
+        """ path to a temp directory for cached initdb. """
         return self._cached_initdb_dir
 
     @cached_initdb_dir.setter
@@ -60,6 +58,7 @@ def cached_initdb_dir(self, value):
 
     @property
     def temp_dir(self):
+        """ path to temp dir containing nodes with default 'base_dir'. """
         return tempfile.tempdir
 
     @temp_dir.setter
@@ -82,6 +81,10 @@ def __setattr__(self, name, value):
         super(GlobalConfig, self).__setattr__(name, value)
 
     def keys(self):
+        """
+        Return a list of all available settings.
+        """
+
         keys = []
 
         for key in dir(GlobalConfig):
@@ -91,15 +94,29 @@ def keys(self):
         return keys
 
     def items(self):
+        """
+        Return setting-value pairs.
+        """
+
         return ((key, self[key]) for key in self.keys())
 
     def update(self, config):
+        """
+        Extract setting-value pairs from 'config' and
+        assign those values to corresponding settings
+        of this GlobalConfig object.
+        """
+
         for key, value in config.items():
             self[key] = value
 
         return self
 
     def copy(self):
+        """
+        Return a copy of this object.
+        """
+
         return copy.copy(self)
 
 
@@ -124,8 +141,8 @@ def _rm_cached_initdb_dirs():
 
 def push_config(**options):
     """
-    Permanently set custom GlobalConfig options
-    and put previous settings on top of stack.
+    Permanently set custom GlobalConfig options and
+    put previous settings on top of the config stack.
     """
 
     # push current config to stack
@@ -150,10 +167,12 @@ def pop_config():
 def scoped_config(**options):
     """
     Temporarily set custom GlobalConfig options for this context.
+    Previous options are pushed to the config stack.
 
     Example:
         >>> from .api import get_new_node
         >>> with scoped_config(cache_initdb=False):
+        ...     # create a new node with fresh initdb
         ...     with get_new_node().init().start() as node:
         ...         print(node.execute('select 1'))
         [(1,)]
@@ -173,7 +192,7 @@ def scoped_config(**options):
 def configure_testgres(**options):
     """
     Adjust current global options.
-    Look at GlobalConfig to learn what can be set.
+    Look at the GlobalConfig to learn about existing settings.
     """
 
     testgres_config.update(options)

testgres/consts.py

@@ -25,5 +25,7 @@
 UTILS_LOG_FILE = "utils.log"
 BACKUP_LOG_FILE = "backup.log"
 
-# default replication slots number
-REPLICATION_SLOTS = 10
+# defaults for node settings
+MAX_REPLICATION_SLOTS = 10
+MAX_WAL_SENDERS = 10
+WAL_KEEP_SEGMENTS = 20

testgres/enums.py

@@ -4,7 +4,7 @@
 
 class XLogMethod(Enum):
     """
-    Available WAL methods for NodeBackup
+    Available WAL methods for :class:`.NodeBackup`
     """
 
     none = 'none'
@@ -14,7 +14,7 @@ class XLogMethod(Enum):
 
 class IsolationLevel(Enum):
     """
-    Transaction isolation level for NodeConnection
+    Transaction isolation level for :class:`.NodeConnection`
     """
 
     ReadUncommitted = 'read uncommitted'
@@ -58,7 +58,6 @@ class ProcessType(Enum):
 
     @staticmethod
     def from_process(process):
-        # yapf: disable
         # legacy names for older releases of PG
         alternative_names = {
             ProcessType.LogicalReplicationLauncher: [
@@ -67,7 +66,7 @@ def from_process(process):
             ProcessType.BackgroundWriter: [
                 'writer'
             ],
-        }
+        }  # yapf: disable
 
         # we deliberately cut special words and spaces
         cmdline = ''.join(process.cmdline()) \