qa: add tests for immutable object cache

based on qemu task, use immutable_object_cache task to test parent cache
based on rbd_fio task, use immutable_object_cache task to test parent cache

Signed-off-by: Yin Congmin <[email protected]>
Signed-off-by: Feng Hualong <[email protected]>

qa/suites/rbd/immutable-object-cache/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/rbd/immutable-object-cache/clusters/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/rbd/immutable-object-cache/clusters/fix-2.yaml

@@ -0,0 +1,3 @@
+roles:
+- [mon.a, mgr.x, osd.0, osd.1]
+- [mon.b, mgr.y, osd.2, osd.3, client.0]

qa/suites/rbd/immutable-object-cache/clusters/openstack.yaml

@@ -0,0 +1,4 @@
+openstack:
+    - volumes: # attached to each instance
+        count: 4
+        size: 10 # GB

qa/suites/rbd/immutable-object-cache/pool/ceph_and_immutable_object_cache.yaml

@@ -0,0 +1,11 @@
+tasks:
+- install:
+    extra_packages: ['ceph-immutable-object-cache']
+- ceph:
+    conf:
+      client:
+        rbd_parent_cache_enabled: true
+        immutable object cache path: /tmp/ceph-immutable-object-cache
+        immutable object cache max size: 10G
+- immutable_object_cache:
+    client.0:

qa/suites/rbd/immutable-object-cache/supported-random-distro$

@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$

qa/suites/rbd/immutable-object-cache/workloads/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/rbd/immutable-object-cache/workloads/fio_on_immutable_object_cache.yaml

@@ -0,0 +1,11 @@
+tasks:
+- rbd_fio:
+    client.0:
+      thick-provision: true
+      fio-io-size: 100%
+      formats: [2]
+      features: [[layering]]
+      io-engine: rbd
+      test-clone-io: 1
+      rw: randread
+      runtime: 600

qa/suites/rbd/immutable-object-cache/workloads/qemu_on_immutable_object_cache_and_thrash.yaml

@@ -0,0 +1,11 @@
+- qemu:
+    client.0:
+      clone: true
+      test: qa/run_xfstests_qemu.sh
+      image_url: http://cloud-images.ubuntu.com/releases/bionic/release/ubuntu-18.04-server-cloudimg-amd64.img
+      cpus: 4
+      memory: 4096
+      disks: 3
+tasks:
+- immutable_object_cache_thrash:
+    client.0:

qa/tasks/immutable_object_cache.py

@@ -0,0 +1,73 @@
+"""
+immutable object cache task
+"""
+from io import StringIO
+
+import contextlib
+import logging
+import os
+import yaml
+import time
+
+from teuthology import misc as teuthology
+from teuthology import contextutil
+from tasks import rbd
+from teuthology.orchestra import run
+from teuthology.config import config as teuth_config
+
+log = logging.getLogger(__name__)
+
+@contextlib.contextmanager
+def immutable_object_cache(ctx, config):
+    """
+    setup and cleanup immutable object cache
+    """
+    log.info("start immutable object cache daemon")
+    for client, client_config in config.items():
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+        # make sure that there is one immutable object cache daemon on the same node.
+        remote.run(
+            args=[
+                'sudo', 'killall', '-s', '9', 'ceph-immutable-object-cache', run.Raw('||'), 'true',
+                ]
+            )
+        remote.run(
+            args=[
+                'ceph-immutable-object-cache', '-b',
+                ]
+            )
+    try:
+        yield
+    finally:
+        log.info("cleanup immutable object cache")
+        for client, client_config in config.items():
+            client_config = client_config if client_config is not None else dict()
+            (remote,) = ctx.cluster.only(client).remotes.keys()
+            remote.run(
+                args=[
+                    'sudo', 'killall', '-s', '9', 'ceph-immutable-object-cache', run.Raw('||'), 'true',
+                    ]
+                )
+            cache_path = client_config.get('immutable object cache path', '/tmp/ceph-immutable-object-cache')
+            remote.run(
+                args=[
+                    'sudo', 'rm', '-rf', cache_path, run.Raw('||'), 'true',
+                    ]
+                )
+
+@contextlib.contextmanager
+def task(ctx, config):
+    """
+    This is task for start immutable_object_cache.
+    """
+    assert isinstance(config, dict), \
+           "task immutable_object_cache only supports a dictionary for configuration"
+
+    managers = []
+    config = teuthology.replace_all_with_clients(ctx.cluster, config)
+    managers.append(
+        lambda: immutable_object_cache(ctx=ctx, config=config)
+        )
+
+    with contextutil.nested(*managers):
+        yield

qa/tasks/immutable_object_cache_thrash.py

@@ -0,0 +1,86 @@
+"""
+immutable object cache thrash task
+"""
+from io import StringIO
+
+import contextlib
+import logging
+import os
+import yaml
+import time
+
+from teuthology import misc as teuthology
+from teuthology import contextutil
+from tasks import rbd
+from teuthology.orchestra import run
+from teuthology.config import config as teuth_config
+
+DEFAULT_KILL_DAEMON_TIME = 2
+DEFAULT_DEAD_TIME = 30
+DEFAULT_LIVE_TIME = 120
+
+log = logging.getLogger(__name__)
+
+@contextlib.contextmanager
+def thrashes_immutable_object_cache_daemon(ctx, config):
+    """
+    thrashes immutable object cache daemon.
+    It can test reconnection feature of RO cache when RO daemon crash
+    TODO : replace sleep with better method.
+    """
+    log.info("thrashes immutable object cache daemon")
+
+    # just thrash one rbd client.
+    client, client_config = list(config.items())[0]
+    (remote,) = ctx.cluster.only(client).remotes.keys()
+    client_config = client_config if client_config is not None else dict()
+    kill_daemon_time = client_config.get('kill_daemon_time', DEFAULT_KILL_DAEMON_TIME)
+    dead_time = client_config.get('dead_time', DEFAULT_DEAD_TIME)
+    live_time = client_config.get('live_time', DEFAULT_LIVE_TIME)
+
+    for i in range(kill_daemon_time):
+        log.info("ceph-immutable-object-cache crash....")
+        remote.run(
+            args=[
+                'sudo', 'killall', '-s', '9', 'ceph-immutable-object-cache', run.Raw('||'), 'true',
+                 ]
+            )
+        # librbd shoud normally run when ceph-immutable-object-cache
+        remote.run(
+            args=[
+                'sleep', '{dead_time}'.format(dead_time=dead_time),
+                 ]
+            )
+        # librbd should reconnect daemon
+        log.info("startup ceph-immutable-object-cache")
+        remote.run(
+            args=[
+                'ceph-immutable-object-cache', '-b',
+                 ]
+            )
+        remote.run(
+            args=[
+                'sleep', '{live_time}'.format(live_time=live_time),
+                 ]
+            )
+    try:
+        yield
+    finally:
+        log.info("cleanup")
+
+@contextlib.contextmanager
+def task(ctx, config):
+    """
+    This is task for testing immutable_object_cache thrash.
+    """
+    assert isinstance(config, dict), \
+            "task immutable_object_cache_thrash only supports a dictionary for configuration"
+
+    managers = []
+    config = teuthology.replace_all_with_clients(ctx.cluster, config)
+    managers.append(
+        lambda: thrashes_immutable_object_cache_daemon(ctx=ctx, config=config)
+        )
+
+    with contextutil.nested(*managers):
+        yield

qa/tasks/rbd_fio.py

@@ -157,6 +157,8 @@ def run_fio(remote, config, rbd_test_dir):
                         '--image', rbd_name,
                         '--image-format', '{f}'.format(f=frmt)]
            map(lambda x: create_args.extend(['--image-feature', x]), feature)
+           if config.get('thick-provision'):
+               create_args.append('--thick-provision')
            remote.run(args=create_args)
            remote.run(args=['rbd', 'info', rbd_name])
            if ioengine != 'rbd':