Merge pull request Aiven-Open#522 from aiven/alex-fix-upload-paths

Fix upload path for basebackup chunks [BF-1161]

pghoard/basebackup.py

@@ -569,6 +569,23 @@ def encryption_data(self) -> EncryptionData:
     def compression_data(self) -> CompressionData:
         return CompressionData.from_config(self.config)
 
+    @staticmethod
+    def chunk_path_to_middle_path_name(chunk_path: Path, file_type: FileType) -> Tuple[Path, str]:
+        chunk_rel_path = chunk_path.relative_to(chunk_path.parent.parent)
+        if file_type == FileType.Basebackup_chunk:
+            middle_path = Path("basebackup_chunk")
+            chunk_name = str(chunk_rel_path)
+        elif file_type == FileType.Basebackup:
+            middle_path = Path("basebackup")
+            chunk_name = chunk_rel_path.name
+        elif file_type == FileType.Basebackup_delta:
+            middle_path = Path("basebackup_delta")
+            chunk_name = chunk_rel_path.name
+        else:
+            raise NotImplementedError(f"Unsupported file type: {file_type}")
+
+        return middle_path, chunk_name
+
     def tar_one_file(
         self,
         *,
@@ -629,18 +646,9 @@ def tar_one_file(
         }
         if extra_metadata:
             metadata.update(extra_metadata)
-        # FIXME: handle the key computation before here
-        chunk_path = Path(chunk_path)
-        base_repo = chunk_path.parent.parent.parent
-        chunk_name = chunk_path.relative_to(base_repo)
-        if file_type == FileType.Basebackup_chunk:
-            middle_path = Path("basebackup_chunk")
-        elif file_type == FileType.Basebackup:
-            middle_path = Path("basebackup")
-            chunk_name = chunk_name.name
-        elif file_type == FileType.Basebackup_delta:
-            middle_path = Path("basebackup_delta")
-            chunk_name = chunk_name.name
+
+        middle_path, chunk_name = PGBaseBackup.chunk_path_to_middle_path_name(Path(chunk_path), file_type)
+
         self.transfer_queue.put(
             UploadEvent(
                 callback_queue=callback_queue,
@@ -654,7 +662,7 @@ def tar_one_file(
         )
 
         # Get the name of the chunk and the name of the parent directory (ie backup "name")
-        return str(chunk_name), input_size, result_size
+        return chunk_name, input_size, result_size
 
     def wait_for_chunk_transfer_to_complete(self, chunk_count, upload_results, chunk_callback_queue, start_time):
         try:

pghoard/wal.py

@@ -214,6 +214,7 @@ def get_current_lsn_from_identify_system(conn_str: str) -> LSN:
     cur = conn.cursor()
     cur.execute("IDENTIFY_SYSTEM")
     sysinfo = cur.fetchone()
+    assert sysinfo
     conn.close()
     assert sysinfo is not None
     return lsn_from_sysinfo(sysinfo, pg_version)

test/test_basebackup.py

@@ -11,6 +11,7 @@
 from copy import deepcopy
 from distutils.version import LooseVersion
 from os import makedirs
+from pathlib import Path
 from queue import Queue
 from subprocess import check_call
 
@@ -20,7 +21,7 @@
 
 from pghoard import common, metrics
 from pghoard.basebackup import PGBaseBackup
-from pghoard.common import BaseBackupMode
+from pghoard.common import BaseBackupMode, FileType
 from pghoard.restore import Restore, RestoreError
 from pghoard.rohmu import get_transfer
 
@@ -807,3 +808,20 @@ def test_patch_basebackup_info(self, pghoard):
         assert entry["metadata"]["backup-reason"] == "requested"
         assert entry["metadata"]["backup-decision-time"] == now - datetime.timedelta(seconds=30)
         assert entry["metadata"]["normalized-backup-time"] is None
+
+    def test_chunk_path_to_middle_path_name(self):
+        assert PGBaseBackup.chunk_path_to_middle_path_name(
+            Path("/a/b/2022-04-19_09-27_0.00000000.pghoard"), FileType.Basebackup
+        ) == (Path("basebackup"), "2022-04-19_09-27_0.00000000.pghoard")
+
+        assert PGBaseBackup.chunk_path_to_middle_path_name(
+            Path("/a/b/2022-04-19_09-27_0/2022-04-19_09-27_0.00000001.pghoard"), FileType.Basebackup_chunk
+        ) == (Path("basebackup_chunk"), "2022-04-19_09-27_0/2022-04-19_09-27_0.00000001.pghoard")
+
+        assert PGBaseBackup.chunk_path_to_middle_path_name(
+            Path("/a/b/0fdc9365aea5447f9a16da8104dc9fcc.delta"), FileType.Basebackup_delta
+        ) == (Path("basebackup_delta"), "0fdc9365aea5447f9a16da8104dc9fcc.delta")
+
+        for file_type in {FileType.Wal, FileType.Metadata, FileType.Timeline}:
+            with pytest.raises(NotImplementedError):
+                assert PGBaseBackup.chunk_path_to_middle_path_name(Path("/a/b/000000010000000000000002"), file_type)