diff --git a/.docs/macros/includes/main.py b/.docs/macros/includes/main.py new file mode 100644 index 000000000000..07f6e6e1cc46 --- /dev/null +++ b/.docs/macros/includes/main.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 + +from pygit2 import Repository +import re + +""" +GitHub branch/tag URL replacer +""" + +regex = r"(github\.com/.+/rook/.+)/master/" +subst = "\\1/%s/" + +def define_env(env): + + repo = Repository('.') + if repo is not None: + target = repo.head.shorthand + + env.variables['current_branch'] = target + +def on_post_page_macros(env): + """ + Replace the branch/tag in the rook GitHub file and directory links pointing to `master` + with the correct one that is currently active. + """ + + target = env.variables['current_branch'] + if target == 'master': + return + + env.raw_markdown = re.sub(regex, subst % target, env.raw_markdown, 0) diff --git a/.docs/overrides/main.html b/.docs/overrides/main.html new file mode 100644 index 000000000000..c5d54bcd2abd --- /dev/null +++ b/.docs/overrides/main.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block outdated %} + This document is for a development version of Rook. + + Click here to go to latest release documentation. + +{% endblock %} diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index c235de7b0fcc..0ceb2cb15503 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -47,6 +47,7 @@ jobs: pip install pylint pip install pylint --upgrade pip install requests + pip install pygit2 - name: Lint Python files run: pylint $(git ls-files '*.py') -E diff --git a/.gitignore b/.gitignore index 4397d0ce16ed..2744899ea639 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ # OLM related stuff deploy/olm/deploy/* deploy/olm/templates/* + +# mkdocs + mike +site/ +public/ +__pycache__/ diff --git a/Documentation/.pages b/Documentation/.pages new file mode 100644 index 000000000000..c6dc6332dbe2 --- /dev/null +++ b/Documentation/.pages @@ -0,0 +1,8 @@ +nav: + - Getting-Started + - Helm-Charts + - Storage-Configuration + - CRDs + - Troubleshooting + - Upgrade + - Contributing diff --git a/Documentation/CRDs/.pages b/Documentation/CRDs/.pages new file mode 100644 index 000000000000..534150cbf859 --- /dev/null +++ b/Documentation/CRDs/.pages @@ -0,0 +1,8 @@ +title: Custom Resources +nav: + - ceph-cluster-crd.md + - Block-Storage + - Shared-Filesystem + - Object-Storage + - ceph-client-crd.md + - ... diff --git a/Documentation/ceph-pool-crd.md b/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md similarity index 88% rename from Documentation/ceph-pool-crd.md rename to Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md index ef13a88c7591..f271093e74ed 100644 --- a/Documentation/ceph-pool-crd.md +++ b/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md @@ -1,23 +1,19 @@ --- title: Block Pool CRD -weight: 2700 -indent: true --- -{% include_relative branch.liquid %} - -# Ceph Block Pool CRD Rook allows creation and customization of storage pools through the custom resource definitions (CRDs). The following settings are available for pools. -## Samples +## Examples ### Replicated For optimal performance, while also adding redundancy, this sample will configure Ceph to make three full copies of the data on multiple nodes. -> **NOTE**: This sample requires *at least 1 OSD per node*, with each OSD located on *3 different nodes*. +!!! note + This sample requires *at least 1 OSD per node*, with each OSD located on *3 different nodes*. -Each OSD must be located on a different node, because the [`failureDomain`](ceph-pool-crd.md#spec) is set to `host` and the `replicated.size` is set to `3`. +Each OSD must be located on a different node, because the [`failureDomain`](ceph-block-pool-crd.md#spec) is set to `host` and the `replicated.size` is set to `3`. ```yaml apiVersion: ceph.rook.io/v1 @@ -33,6 +29,7 @@ spec: ``` #### Hybrid Storage Pools + Hybrid storage is a combination of two different storage tiers. For example, SSD and HDD. This helps to improve the read performance of cluster by placing, say, 1st copy of data on the higher performance tier (SSD or NVME) and remaining replicated copies on lower cost tier (HDDs). @@ -54,15 +51,18 @@ spec: primaryDeviceClass: ssd secondaryDeviceClass: hdd ``` -> **IMPORTANT**: The device classes `primaryDeviceClass` and `secondaryDeviceClass` must have at least one OSD associated with them or else the pool creation will fail. + +!!! important + The device classes `primaryDeviceClass` and `secondaryDeviceClass` must have at least one OSD associated with them or else the pool creation will fail. ### Erasure Coded This sample will lower the overall storage capacity requirement, while also adding redundancy by using [erasure coding](#erasure-coding). -> **NOTE**: This sample requires *at least 3 bluestore OSDs*. +!!! note + This sample requires *at least 3 bluestore OSDs*. -The OSDs can be located on a single Ceph node or spread across multiple nodes, because the [`failureDomain`](ceph-pool-crd.md#spec) is set to `osd` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). +The OSDs can be located on a single Ceph node or spread across multiple nodes, because the [`failureDomain`](ceph-block-pool-crd.md#spec) is set to `osd` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). ```yaml apiVersion: ceph.rook.io/v1 @@ -81,7 +81,7 @@ spec: High performance applications typically will not use erasure coding due to the performance overhead of creating and distributing the chunks in the cluster. When creating an erasure-coded pool, it is highly recommended to create the pool when you have **bluestore OSDs** in your cluster -(see the [OSD configuration settings](ceph-cluster-crd.md#osd-configuration-settings). Filestore OSDs have +(see the [OSD configuration settings](../ceph-cluster-crd.md#osd-configuration-settings). Filestore OSDs have [limitations](http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites) that are unsafe and lower performance. ### Mirroring @@ -124,10 +124,8 @@ This secret can then be fetched like so: ```console kubectl get secret -n rook-ceph pool-peer-token-replicapool -o jsonpath='{.data.token}'|base64 -d +eyJmc2lkIjoiOTFlYWUwZGQtMDZiMS00ZDJjLTkxZjMtMTMxMWM5ZGYzODJiIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFEN1psOWZ3V1VGRHhBQWdmY0gyZi8xeUhYeGZDUTU5L1N0NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjEwLjEwMS4xOC4yMjM6MzMwMCx2MToxMC4xMDEuMTguMjIzOjY3ODldIn0= ``` ->``` ->eyJmc2lkIjoiOTFlYWUwZGQtMDZiMS00ZDJjLTkxZjMtMTMxMWM5ZGYzODJiIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFEN1psOWZ3V1VGRHhBQWdmY0gyZi8xeUhYeGZDUTU5L1N0NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjEwLjEwMS4xOC4yMjM6MzMwMCx2MToxMC4xMDEuMTguMjIzOjY3ODldIn0= ->``` The secret must be decoded. The result will be another base64 encoded blob that you will import in the destination cluster: @@ -193,12 +191,13 @@ stretched) then you will have 2 replicas per datacenter where each replica ends * `erasureCoded`: Settings for an erasure-coded pool. If specified, `replicated` settings must not be specified. See below for more details on [erasure coding](#erasure-coding). * `dataChunks`: Number of chunks to divide the original object into * `codingChunks`: Number of coding chunks to generate -* `failureDomain`: The failure domain across which the data will be spread. This can be set to a value of either `osd` or `host`, with `host` being the default setting. A failure domain can also be set to a different type (e.g. `rack`), if the OSDs are created on nodes with the supported [topology labels](ceph-cluster-crd.md#osd-topology). If the `failureDomain` is changed on the pool, the operator will create a new CRUSH rule and update the pool. +* `failureDomain`: The failure domain across which the data will be spread. This can be set to a value of either `osd` or `host`, with `host` being the default setting. A failure domain can also be set to a different type (e.g. `rack`), if the OSDs are created on nodes with the supported [topology labels](../ceph-cluster-crd.md#osd-topology). If the `failureDomain` is changed on the pool, the operator will create a new CRUSH rule and update the pool. If a `replicated` pool of size `3` is configured and the `failureDomain` is set to `host`, all three copies of the replicated data will be placed on OSDs located on `3` different Ceph hosts. This case is guaranteed to tolerate a failure of two hosts without a loss of data. Similarly, a failure domain set to `osd`, can tolerate a loss of two OSD devices. If erasure coding is used, the data and coding chunks are spread across the configured failure domain. - > **NOTE**: Neither Rook, nor Ceph, prevent the creation of a cluster where the replicated data (or Erasure Coded chunks) can be written safely. By design, Ceph will delay checking for suitable OSDs until a write request is made and this write can hang if there are not sufficient OSDs to satisfy the request. + !!! caution + Neither Rook, nor Ceph, prevent the creation of a cluster where the replicated data (or Erasure Coded chunks) can be written safely. By design, Ceph will delay checking for suitable OSDs until a write request is made and this write can hang if there are not sufficient OSDs to satisfy the request. * `deviceClass`: Sets up the CRUSH rule for the pool to distribute data only on the specified device class. If left empty or unspecified, the pool will use the cluster's default CRUSH root, which usually distributes data over all OSDs, regardless of their class. * `crushRoot`: The root in the crush map to be used by the pool. If left empty or unspecified, the default root will be used. Creating a crush hierarchy for the OSDs currently requires the Rook toolbox to run the Ceph tools described [here](http://docs.ceph.com/docs/master/rados/operations/crush-map/#modifying-the-crush-map). * `enableRBDStats`: Enables collecting RBD per-image IO statistics by enabling dynamic OSD performance counters. Defaults to false. For more info see the [ceph documentation](https://docs.ceph.com/docs/master/mgr/prometheus/#rbd-io-statistics). @@ -206,7 +205,7 @@ stretched) then you will have 2 replicas per datacenter where each replica ends require names that are incompatible with K8s resource names. These special pools can be configured by setting this `name` to override the name of the Ceph pool that is created instead of using the `metadata.name` for the pool. Only the following pool names are supported: `device_health_metrics`, `.nfs`, and `.mgr`. See the example - [builtin mgr pool](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/pool-builtin-mgr.yaml). + [builtin mgr pool](https://github.com/rook/rook/blob/master/deploy/examples/pool-builtin-mgr.yaml). * `parameters`: Sets any [parameters](https://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values) listed to the given pool * `target_size_ratio:` gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool, for more info see the [ceph documentation](https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size) @@ -229,7 +228,9 @@ stretched) then you will have 2 replicas per datacenter where each replica ends * `quotas`: Set byte and object quotas. See the [ceph documentation](https://docs.ceph.com/en/latest/rados/operations/pools/#set-pool-quotas) for more info. * `maxSize`: quota in bytes as a string with quantity suffixes (e.g. "10Gi") * `maxObjects`: quota in objects as an integer - > **NOTE**: A value of 0 disables the quota. + + !!! note + A value of 0 disables the quota. ### Add specific pool properties @@ -273,4 +274,4 @@ The `failureDomain` must be also be taken into account when determining the numb If you do not have a sufficient number of hosts or OSDs for unique placement the pool can be created, writing to the pool will hang. -Rook currently only configures two levels in the CRUSH map. It is also possible to configure other levels such as `rack` with by adding [topology labels](ceph-cluster-crd.md#osd-topology) to the nodes. +Rook currently only configures two levels in the CRUSH map. It is also possible to configure other levels such as `rack` with by adding [topology labels](../ceph-cluster-crd.md#osd-topology) to the nodes. diff --git a/Documentation/ceph-pool-radosnamespace.md b/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md similarity index 91% rename from Documentation/ceph-pool-radosnamespace.md rename to Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md index daa56823d73a..371ad50d46de 100644 --- a/Documentation/ceph-pool-radosnamespace.md +++ b/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md @@ -1,14 +1,8 @@ --- -title: RADOS Namespace CRD -weight: 3610 -indent: true +title: Block Pool RADOS Namespace CRD --- -{% include_relative branch.liquid %} - -This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](quickstart.md) - -# CephBlockPoolRadosNamespace CRD +This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](../../Getting-Started/quickstart.md) RADOS currently uses pools both for data distribution (pools are shared into PGs, which map to OSDs) and as the granularity for security (capabilities can diff --git a/Documentation/ceph-rbd-mirror-crd.md b/Documentation/CRDs/Block-Storage/ceph-rbd-mirror-crd.md similarity index 86% rename from Documentation/ceph-rbd-mirror-crd.md rename to Documentation/CRDs/Block-Storage/ceph-rbd-mirror-crd.md index eb228923df04..315dc3148cfc 100644 --- a/Documentation/ceph-rbd-mirror-crd.md +++ b/Documentation/CRDs/Block-Storage/ceph-rbd-mirror-crd.md @@ -1,11 +1,6 @@ --- title: RBD Mirror CRD -weight: 3500 -indent: true --- -{% include_relative branch.liquid %} - -# Ceph RBDMirror CRD Rook allows creation and updating rbd-mirror daemon(s) through the custom resource definitions (CRDs). RBD images can be asynchronously mirrored between two Ceph clusters. @@ -27,7 +22,7 @@ spec: ### Prerequisites -This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](quickstart.md) +This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](../../Getting-Started/quickstart.md) ## Settings @@ -41,7 +36,7 @@ If any setting is unspecified, a suitable default will be used automatically. ### RBDMirror Settings * `count`: The number of rbd mirror instance to run. -* `placement`: The rbd mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml).. +* `placement`: The rbd mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml).. * `annotations`: Key value pair list of annotations to add. * `labels`: Key value pair list of labels to add. * `resources`: The resource requirements for the rbd mirror pods. @@ -50,4 +45,4 @@ If any setting is unspecified, a suitable default will be used automatically. ### Configuring mirroring peers Configure mirroring peers individually for each CephBlockPool. Refer to the -[CephBlockPool documentation](ceph-pool-crd.md#mirroring) for more detail. +[CephBlockPool documentation](ceph-block-pool-crd.md#mirroring) for more detail. diff --git a/Documentation/CRDs/Object-Storage/.pages b/Documentation/CRDs/Object-Storage/.pages new file mode 100644 index 000000000000..b669e3376493 --- /dev/null +++ b/Documentation/CRDs/Object-Storage/.pages @@ -0,0 +1,4 @@ +nav: + - ceph-object-store-crd.md + - ceph-object-store-user-crd.md + - ... diff --git a/Documentation/ceph-object-multisite-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-multisite-crd.md similarity index 88% rename from Documentation/ceph-object-multisite-crd.md rename to Documentation/CRDs/Object-Storage/ceph-object-multisite-crd.md index dff60592021f..f5d4373e2ecd 100644 --- a/Documentation/ceph-object-multisite-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-multisite-crd.md @@ -1,18 +1,14 @@ --- title: Object Multisite CRDs -weight: 2825 -indent: true --- -# Ceph Object Multisite CRDs - -The following CRDs enable Ceph object stores to isolate or replicate data via multisite. For more information on multisite, visit the [ceph-object-multisite](/Documentation/ceph-object-multisite.md) documentation. +The following CRDs enable Ceph object stores to isolate or replicate data via multisite. For more information on multisite, visit the [Ceph Object Multisite CRDs documentation](../../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md). ## Ceph Object Realm CRD Rook allows creation of a realm in a ceph cluster for object stores through the custom resource definitions (CRDs). The following settings are available for Ceph object store realms. -### Sample +### Example ```yaml apiVersion: ceph.rook.io/v1 @@ -42,7 +38,7 @@ spec: Rook allows creation of zone groups in a ceph cluster for object stores through the custom resource definitions (CRDs). The following settings are available for Ceph object store zone groups. -### Sample +### Example ```yaml apiVersion: ceph.rook.io/v1 @@ -69,7 +65,7 @@ spec: Rook allows creation of zones in a ceph cluster for object stores through the custom resource definitions (CRDs). The following settings are available for Ceph object store zone. -### Sample +### Example ```yaml apiVersion: ceph.rook.io/v1 @@ -99,7 +95,7 @@ spec: ### Pools -The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](ceph-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least three devices (2 data + 1 coding chunks) in the cluster. +The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](../Block-Storage/ceph-block-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least three devices (2 data + 1 coding chunks) in the cluster. #### Spec diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md similarity index 89% rename from Documentation/ceph-object-store-crd.md rename to Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 575190426877..1105dff534d8 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -1,22 +1,19 @@ --- title: Object Store CRD -weight: 2800 -indent: true --- -# Ceph Object Store CRD - Rook allows creation and customization of object stores through the custom resource definitions (CRDs). The following settings are available for Ceph object stores. -## Sample +## Example ### Erasure Coded Erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool. -> **NOTE**: This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. +!!! note + This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. -The OSDs must be located on different nodes, because the [`failureDomain`](ceph-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). +The OSDs must be located on different nodes, because the [`failureDomain`](../Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). ```yaml apiVersion: ceph.rook.io/v1 @@ -79,7 +76,7 @@ spec: ### Pools -The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](ceph-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least three devices (2 data + 1 coding chunks) in the cluster. +The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](../Block-Storage/ceph-block-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least three devices (2 data + 1 coding chunks) in the cluster. When the `zone` section is set pools with the object stores name will not be created since the object-store will the using the pools created by the ceph-object-zone. @@ -110,13 +107,14 @@ The gateway settings correspond to the RGW daemon settings. * `port`: The port on which the Object service will be reachable. If host networking is enabled, the RGW daemons will also listen on that port. If running on SDN, the RGW daemon listening port will be 8080 internally. * `securePort`: The secure port on which RGW pods will be listening. A TLS certificate must be specified either via `sslCerticateRef` or `service.annotations` * `instances`: The number of pods that will be started to load balance this object store. -* `externalRgwEndpoints`: A list of IP addresses to connect to external existing Rados Gateways (works with external mode). This setting will be ignored if the `CephCluster` does not have `external` spec enabled. Refer to the [external cluster section](ceph-cluster-crd.md#external-cluster) for more details. +* `externalRgwEndpoints`: A list of IP addresses to connect to external existing Rados Gateways (works with external mode). This setting will be ignored if the `CephCluster` does not have `external` spec enabled. Refer to the [external cluster section](../ceph-cluster-crd.md#external-cluster) for more details. * `annotations`: Key value pair list of annotations to add. * `labels`: Key value pair list of labels to add. * `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster. -* `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits). +* `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](../ceph-cluster-crd.md#resource-requirementslimits). * `priorityClassName`: Set priority class name for the Gateway Pod(s) * `service`: The annotations to set on to the Kubernetes Service of RGW. The [service serving cert](https://docs.openshift.com/container-platform/4.6/security/certificates/service-serving-certificate.html) feature supported in Openshift is enabled by the following example: + ```yaml gateway: service: @@ -138,7 +136,7 @@ All the other settings from the gateway section will be ignored, except for `sec ## Zone Settings -The [zone](ceph-object-multisite.md) settings allow the object store to join custom created [ceph-object-zone](ceph-object-multisite-crd.md). +The [zone](../../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) settings allow the object store to join custom created [ceph-object-zone](ceph-object-multisite-crd.md). * `name`: the name of the ceph-object-zone the object store will be in. @@ -202,7 +200,7 @@ Rook-Ceph always keeps the bucket and the user for the health check, it just doe ## Security settings -Ceph RGW supports encryption via Key Management System (KMS) using HashiCorp Vault. Refer to the [vault kms section](ceph-cluster-crd.md#vault-kms) for detailed explanation. +Ceph RGW supports encryption via Key Management System (KMS) using HashiCorp Vault. Refer to the [vault kms section](../ceph-cluster-crd.md#vault-kms) for detailed explanation. If these settings are defined, then RGW establish a connection between Vault and whenever S3 client sends a request with Server Side Encryption, it encrypts that using the key specified by the client. For more details w.r.t RGW, please refer [Ceph Vault documentation](https://docs.ceph.com/en/latest/radosgw/vault/) @@ -225,8 +223,11 @@ For RGW, please note the following: * `VAULT_SECRET_ENGINE` option is specifically for RGW to mention about the secret engine which can be used, currently supports two: [kv](https://www.vaultproject.io/docs/secrets/kv) and [transit](https://www.vaultproject.io/docs/secrets/transit). And for kv engine only version 2 is supported. * The Storage administrator needs to create a secret in the Vault server so that S3 clients use that key for encryption -$ vault kv put rook/ key=$(openssl rand -base64 32) # kv engine -$ vault write -f transit/keys/ exportable=true # transit engine + +```console +vault kv put rook/ key=$(openssl rand -base64 32) # kv engine +vault write -f transit/keys/ exportable=true # transit engine +``` * TLS authentication with custom certificates between Vault and CephObjectStore RGWs are supported from ceph v16.2.6 onwards @@ -242,6 +243,7 @@ manual deletion or removal of all ObjectBucketClaims. Alternately, the Kubernetes Custom Resource, but the Ceph pools which store the data will not be removed in this case. Rook will warn about which buckets are blocking deletion in three ways: + 1. An event will be registered on the CephObjectStore resource 1. A status condition will be added to the CephObjectStore resource 1. An error will be added to the Rook-Ceph Operator log diff --git a/Documentation/ceph-object-store-user-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md similarity index 65% rename from Documentation/ceph-object-store-user-crd.md rename to Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md index 27bbdbffaf3f..8addf84b68ce 100644 --- a/Documentation/ceph-object-store-user-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md @@ -1,15 +1,11 @@ --- title: Object Store User CRD -weight: 2900 -indent: true --- -# Ceph Object Store User CRD - Rook allows creation and customization of object store users through the custom resource definitions (CRDs). The following settings are available for Ceph object store users. -## Sample +## Example ```yaml apiVersion: ceph.rook.io/v1 @@ -40,16 +36,15 @@ spec: * `store`: The object store in which the user will be created. This matches the name of the objectstore CRD. * `displayName`: The display name which will be passed to the `radosgw-admin user create` command. -* `quotas`: This represents quota limitation can be set on the user (support added in Rook v1.7.3 and up). - Please refer [here](https://docs.ceph.com/en/latest/radosgw/admin/#quota-management) for details. - * `maxBuckets`: The maximum bucket limit for the user. - * `maxSize`: Maximum size limit of all objects across all the user's buckets. - * `maxObjects`: Maximum number of objects across all the user's buckets. +* `quotas`: This represents quota limitation can be set on the user (support added in Rook v1.7.3 and up). Please refer [here](https://docs.ceph.com/en/latest/radosgw/admin/#quota-management) for details. + * `maxBuckets`: The maximum bucket limit for the user. + * `maxSize`: Maximum size limit of all objects across all the user's buckets. + * `maxObjects`: Maximum number of objects across all the user's buckets. * `capabilities`: Ceph allows users to be given additional permissions (support added in Rook v1.7.3 and up). Due to missing APIs in go-ceph for updating the user capabilities, this setting can currently only be used during the creation of the object store user. If a user's capabilities need modified, the user must be deleted and re-created. - See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/#add-remove-admin-capabilities) for more info. - Rook supports adding `read`, `write`, `read, write`, or `*` permissions for the following resources: - * `users` - * `buckets` - * `usage` - * `metadata` - * `zone` + See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/#add-remove-admin-capabilities) for more info. + Rook supports adding `read`, `write`, `read, write`, or `*` permissions for the following resources: + * `users` + * `buckets` + * `usage` + * `metadata` + * `zone` diff --git a/Documentation/ceph-filesystem-crd.md b/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md similarity index 83% rename from Documentation/ceph-filesystem-crd.md rename to Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md index a883b9065126..c99124ae9b57 100644 --- a/Documentation/ceph-filesystem-crd.md +++ b/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md @@ -1,23 +1,19 @@ --- -title: Shared Filesystem CRD -weight: 3000 -indent: true +title: Filesystem CRD --- -{% include_relative branch.liquid %} - -# Ceph Shared Filesystem CRD Rook allows creation and customization of shared filesystems through the custom resource definitions (CRDs). The following settings are available for Ceph filesystems. -## Samples +## Examples ### Replicated -> **NOTE**: This sample requires *at least 1 OSD per node*, with each OSD located on *3 different nodes*. +!!! note + This sample requires *at least 1 OSD per node*, with each OSD located on *3 different nodes*. -Each OSD must be located on a different node, because both of the defined pools set the [`failureDomain`](ceph-pool-crd.md#spec) to `host` and the `replicated.size` to `3`. +Each OSD must be located on a different node, because both of the defined pools set the [`failureDomain`](../Block-Storage/ceph-block-pool-crd.md#spec) to `host` and the `replicated.size` to `3`. -The `failureDomain` can also be set to another location type (e.g. `rack`), if it has been added as a `location` in the [Storage Selection Settings](ceph-cluster-crd.md#storage-selection-settings). +The `failureDomain` can also be set to another location type (e.g. `rack`), if it has been added as a `location` in the [Storage Selection Settings](../ceph-cluster-crd.md#storage-selection-settings). ```yaml apiVersion: ceph.rook.io/v1 @@ -66,15 +62,15 @@ spec: # memory: "1024Mi" ``` -(These definitions can also be found in the [`filesystem.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/filesystem.yaml) file) +(These definitions can also be found in the [`filesystem.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/filesystem.yaml) file) ### Erasure Coded -Erasure coded pools require the OSDs to use `bluestore` for the configured [`storeType`](ceph-cluster-crd.md#osd-configuration-settings). Additionally, erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool. - -> **NOTE**: This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. +Erasure coded pools require the OSDs to use `bluestore` for the configured [`storeType`](../ceph-cluster-crd.md#osd-configuration-settings). Additionally, erasure coded pools can only be used with `dataPools`. The `metadataPool` must use a replicated pool. +!!! note + This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. -The OSDs must be located on different nodes, because the [`failureDomain`](ceph-pool-crd.md#spec) will be set to `host` by default, and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). +The OSDs must be located on different nodes, because the [`failureDomain`](../Block-Storage/ceph-block-pool-crd.md#spec) will be set to `host` by default, and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). ```yaml apiVersion: ceph.rook.io/v1 @@ -101,8 +97,8 @@ spec: **IMPORTANT**: For erasure coded pools, we have to create a replicated pool as the default data pool and an erasure-coded pool as a secondary pool. -(These definitions can also be found in the [`filesystem-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/filesystem-ec.yaml) file. -Also see an example in the [`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/csi/cephfs/storageclass-ec.yaml) for how to configure the volume.) +(These definitions can also be found in the [`filesystem-ec.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/filesystem-ec.yaml) file. +Also see an example in the [`storageclass-ec.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/csi/cephfs/storageclass-ec.yaml) for how to configure the volume.) ### Mirroring @@ -165,11 +161,9 @@ status: This secret can then be fetched like so: ```console -kubectl get secret -n rook-ceph fs-peer-token-myfs -o jsonpath='{.data.token}'|base64 -d +$ kubectl get secret -n rook-ceph fs-peer-token-myfs -o jsonpath='{.data.token}'|base64 -d +eyJmc2lkIjoiOTFlYWUwZGQtMDZiMS00ZDJjLTkxZjMtMTMxMWM5ZGYzODJiIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFEN1psOWZ3V1VGRHhBQWdmY0gyZi8xeUhYeGZDUTU5L1N0NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjEwLjEwMS4xOC4yMjM6MzMwMCx2MToxMC4xMDEuMTguMjIzOjY3ODldIn0= ``` ->``` ->eyJmc2lkIjoiOTFlYWUwZGQtMDZiMS00ZDJjLTkxZjMtMTMxMWM5ZGYzODJiIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFEN1psOWZ3V1VGRHhBQWdmY0gyZi8xeUhYeGZDUTU5L1N0NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjEwLjEwMS4xOC4yMjM6MzMwMCx2MToxMC4xMDEuMTguMjIzOjY3ODldIn0= ->``` The secret must be decoded. The result will be another base64 encoded blob that you will import in the destination cluster: @@ -188,7 +182,7 @@ See the official cephfs mirror documentation on [how to add a bootstrap peer](ht ### Pools -The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](ceph-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least eight devices (6 data + 2 coding chunks) in the cluster. +The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](../Block-Storage/ceph-block-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least eight devices (6 data + 2 coding chunks) in the cluster. * `metadataPool`: The settings used to create the filesystem metadata pool. Must use replication. * `dataPools`: The settings to create the filesystem data pools. Optionally (and we highly recommend), a pool name can be specified with the `name` field to override the default generated name; see more below. If multiple pools are specified, Rook will add the pools to the filesystem. Assigning users or files to a pool is left as an exercise for the reader with the [CephFS documentation](http://docs.ceph.com/docs/master/cephfs/file-layouts/). The data pools can use replication or erasure coding. If erasure coding pools are specified, the cluster must be running with bluestore enabled on the OSDs. @@ -220,7 +214,7 @@ The metadata server settings correspond to the MDS daemon settings. * `duration`: * `annotations`: Key value pair list of annotations to add. * `labels`: Key value pair list of labels to add. -* `placement`: The mds pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml). +* `placement`: The mds pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml). * `resources`: Set resource requests/limits for the Filesystem MDS Pod(s), see [MDS Resources Configuration Settings](#mds-resources-configuration-settings) * `priorityClassName`: Set priority class name for the Filesystem MDS Pod(s) * `startupProbe` : Disable, or override timing and threshold values of the Filesystem MDS startup probe @@ -228,7 +222,7 @@ The metadata server settings correspond to the MDS daemon settings. ### MDS Resources Configuration Settings -The format of the resource requests/limits structure is the same as described in the [Ceph Cluster CRD documentation](ceph-cluster-crd.md#resource-requirementslimits). +The format of the resource requests/limits structure is the same as described in the [Ceph Cluster CRD documentation](../ceph-cluster-crd.md#resource-requirementslimits). If the memory resource limit is declared Rook will automatically set the MDS configuration `mds_cache_memory_limit`. The configuration value is calculated with the aim that the actual MDS memory consumption remains consistent with the MDS pods' resource declaration. diff --git a/Documentation/ceph-fs-mirror-crd.md b/Documentation/CRDs/Shared-Filesystem/ceph-fs-mirror-crd.md similarity index 90% rename from Documentation/ceph-fs-mirror-crd.md rename to Documentation/CRDs/Shared-Filesystem/ceph-fs-mirror-crd.md index 68bc8c430871..786b5e64faaa 100644 --- a/Documentation/ceph-fs-mirror-crd.md +++ b/Documentation/CRDs/Shared-Filesystem/ceph-fs-mirror-crd.md @@ -1,14 +1,8 @@ --- title: Filesystem Mirror CRD -weight: 3600 -indent: true --- -{% include_relative branch.liquid %} - -This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](quickstart.md) - -# Ceph FilesystemMirror CRD +This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](../../Getting-Started/quickstart.md) Rook allows creation and updating the fs-mirror daemon through the custom resource definitions (CRDs). CephFS will support asynchronous replication of snapshots to a remote (different Ceph cluster) CephFS file system via cephfs-mirror tool. @@ -38,7 +32,7 @@ If any setting is unspecified, a suitable default will be used automatically. ### FilesystemMirror Settings -- `placement`: The cephfs-mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml). +- `placement`: The cephfs-mirror pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml). - `annotations`: Key value pair list of annotations to add. - `labels`: Key value pair list of labels to add. - `resources`: The resource requirements for the cephfs-mirror pods. diff --git a/Documentation/ceph-fs-subvolumegroup.md b/Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md similarity index 84% rename from Documentation/ceph-fs-subvolumegroup.md rename to Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md index b432bf636c80..a53c7709317d 100644 --- a/Documentation/ceph-fs-subvolumegroup.md +++ b/Documentation/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd.md @@ -1,14 +1,9 @@ --- -title: SubVolume Group CRD -weight: 3610 -indent: true +title: Filesystem SubVolume Group CRD --- -{% include_relative branch.liquid %} - -This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](quickstart.md) - -# CephFilesystemSubVolumeGroup CRD +!!! info + This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](../../Getting-Started/quickstart.md) Rook allows creation of Ceph Filesystem [SubVolumeGroups](https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-subvolume-groups) through the custom resource definitions (CRDs). Filesystem subvolume groups are an abstraction for a directory level higher than Filesystem subvolumes to effect policies (e.g., File layouts) across a set of subvolumes. diff --git a/Documentation/ceph-client-crd.md b/Documentation/CRDs/ceph-client-crd.md similarity index 93% rename from Documentation/ceph-client-crd.md rename to Documentation/CRDs/ceph-client-crd.md index 7a8fccd67b63..9f16a55d6e41 100644 --- a/Documentation/ceph-client-crd.md +++ b/Documentation/CRDs/ceph-client-crd.md @@ -1,11 +1,7 @@ --- title: Client CRD -weight: 3500 -indent: true --- -# Ceph Client CRD - Rook allows creation and updating clients through the custom resource definitions (CRDs). For more information about user management and capabilities see the [Ceph docs](https://docs.ceph.com/docs/master/rados/operations/user-management/). @@ -44,4 +40,4 @@ spec: ### Prerequisites -This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](quickstart.md) +This guide assumes you have created a Rook cluster as explained in the main [Quickstart guide](../Getting-Started/quickstart.md). diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/CRDs/ceph-cluster-crd.md similarity index 90% rename from Documentation/ceph-cluster-crd.md rename to Documentation/CRDs/ceph-cluster-crd.md index 3967a9c7e947..0db70b2790ca 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/CRDs/ceph-cluster-crd.md @@ -1,11 +1,7 @@ --- title: Cluster CRD -weight: 2600 -indent: true --- -# Ceph Cluster CRD - Rook allows creation and customization of storage clusters through the custom resource definitions (CRDs). There are primarily three different modes in which to create your cluster. @@ -20,8 +16,9 @@ Following is an example for each of these approaches. More examples are included To get you started, here is a simple example of a CRD to configure a Ceph cluster with all nodes and all devices. The Ceph persistent data is stored directly on a host path (Ceph Mons) and on raw devices (Ceph OSDs). -> **NOTE**: In addition to your CephCluster object, you need to create the namespace, service accounts, and RBAC rules for the namespace you are going to create the CephCluster in. -> These resources are defined in the example `common.yaml`. +!!! note + In addition to your CephCluster object, you need to create the namespace, service accounts, and RBAC rules for the namespace you are going to create the CephCluster in. + These resources are defined in the example `common.yaml`. ```yaml apiVersion: ceph.rook.io/v1 @@ -175,14 +172,14 @@ Settings can be specified at the global level to apply to the cluster as a whole * On **Minikube** environments, use `/data/rook`. Minikube boots into a tmpfs but it provides some [directories](https://github.com/kubernetes/minikube/blob/master/site/content/en/docs/handbook/persistent_volumes.md#a-note-on-mounts-persistence-and-minikube-hosts) where files can be persisted across reboots. Using one of these directories will ensure that Rook's data and configuration files are persisted and that enough storage space is available. * **WARNING**: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by `dataDirHostPath` must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start. If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir). -* `skipUpgradeChecks`: if set to true Rook won't perform any upgrade checks on Ceph daemons during an upgrade. Use this at **YOUR OWN RISK**, only if you know what you're doing. To understand Rook's upgrade process of Ceph, read the [upgrade doc](ceph-upgrade.md#ceph-version-upgrades). +* `skipUpgradeChecks`: if set to true Rook won't perform any upgrade checks on Ceph daemons during an upgrade. Use this at **YOUR OWN RISK**, only if you know what you're doing. To understand Rook's upgrade process of Ceph, read the [upgrade doc](../Upgrade/rook-upgrade.md#ceph-version-upgrades). * `continueUpgradeAfterChecksEvenIfNotHealthy`: if set to true Rook will continue the OSD daemon upgrade process even if the PGs are not clean, or continue with the MDS upgrade even the file system is not healthy. -* `dashboard`: Settings for the Ceph dashboard. To view the dashboard in your browser see the [dashboard guide](ceph-dashboard.md). +* `dashboard`: Settings for the Ceph dashboard. To view the dashboard in your browser see the [dashboard guide](../Storage-Configuration/Monitoring/ceph-dashboard.md). * `enabled`: Whether to enable the dashboard to view cluster status * `urlPrefix`: Allows to serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy) * `port`: Allows to change the default port where the dashboard is served * `ssl`: Whether to serve the dashboard via SSL, ignored on Ceph versions older than `13.2.2` -* `monitoring`: Settings for monitoring Ceph using Prometheus. To enable monitoring on your cluster see the [monitoring guide](ceph-monitoring.md#prometheus-alerts). +* `monitoring`: Settings for monitoring Ceph using Prometheus. To enable monitoring on your cluster see the [monitoring guide](../Storage-Configuration/Monitoring/ceph-monitoring.md#prometheus-alerts). * `enabled`: Whether to enable prometheus based monitoring for this cluster * `externalMgrEndpoints`: external cluster manager endpoints * `externalMgrPrometheusPort`: external prometheus manager module port. See [external cluster configuration](#external-cluster) for more details. @@ -192,7 +189,7 @@ If this value is empty, each pod will get an ephemeral directory to store their * If you have multiple Rook Ceph clusters in the same Kubernetes cluster, choose the same namespace to set `rulesNamespace` for all the clusters (ideally, namespace with prometheus deployed). Otherwise, you will get duplicate alerts with duplicate alert definitions. * `network`: For the network settings for the cluster, refer to the [network configuration settings](#network-configuration-settings) * `mon`: contains mon related options [mon settings](#mon-settings) -For more details on the mons and when to choose a number other than `3`, see the [mon health doc](ceph-mon-health.md). +For more details on the mons and when to choose a number other than `3`, see the [mon health doc](../Storage-Configuration/Advanced/ceph-mon-health.md). * `mgr`: manager top level section * `count`: set number of ceph managers between `1` to `2`. The default value is 2. If there are two managers, it is important for all mgr services point to the active mgr and not the passive mgr. Therefore, Rook will @@ -220,8 +217,8 @@ For more details on the mons and when to choose a number other than `3`, see the * [storage selection settings](#storage-selection-settings) * [Storage Class Device Sets](#storage-class-device-sets) * `onlyApplyOSDPlacement`: Whether the placement specific for OSDs is merged with the `all` placement. If `false`, the OSD placement will be merged with the `all` placement. If true, the `OSD placement will be applied` and the `all` placement will be ignored. The placement for OSDs is computed from several different places depending on the type of OSD: - - For non-PVCs: `placement.all` and `placement.osd` - - For PVCs: `placement.all` and inside the storageClassDeviceSets from the `placement` or `preparePlacement` + * For non-PVCs: `placement.all` and `placement.osd` + * For PVCs: `placement.all` and inside the storageClassDeviceSets from the `placement` or `preparePlacement` * `disruptionManagement`: The section for configuring management of daemon disruptions * `managePodBudgets`: if `true`, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically via the strategy outlined in the [design](https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md). The operator will block eviction of OSDs by default and unblock them safely when drains are detected. * `osdMaintenanceTimeout`: is a duration in minutes that determines how long an entire failureDomain like `region/zone/host` will be held in `noout` (in addition to the default DOWN/OUT interval) when it is draining. This is only relevant when `managePodBudgets` is `true`. The default value is `30` minutes. @@ -229,7 +226,7 @@ For more details on the mons and when to choose a number other than `3`, see the * `machineDisruptionBudgetNamespace`: the namespace in which to watch the MachineDisruptionBudgets. * `removeOSDsIfOutAndSafeToRemove`: If `true` the operator will remove the OSDs that are down and whose data has been restored to other OSDs. In Ceph terms, the OSDs are `out` and `safe-to-destroy` when they are removed. * `cleanupPolicy`: [cleanup policy settings](#cleanup-policy) -* `security`: [security page for key management configuration](ceph-kms.md) +* `security`: [security page for key management configuration](../Storage-Configuration/Advanced/key-management-system.md) ### Ceph container images @@ -254,7 +251,7 @@ A specific will contain a specific release of Ceph as well as security fixes fro For higher durability in case of mon loss, an even number can be specified although availability may be lower. To maintain quorum a majority of mons must be up. For example, if there are three mons, two must be up. If there are four mons, three must be up. If there are two mons, both must be up. - If quorum is lost, see the [disaster recovery guide](ceph-disaster-recovery.md#restoring-mon-quorum) to restore quorum from a single mon. + If quorum is lost, see the [disaster recovery guide](../Troubleshooting/disaster-recovery.md#restoring-mon-quorum) to restore quorum from a single mon. * `allowMultiplePerNode`: Whether to allow the placement of multiple mons on a single node. Default is `false` for production. Should only be set to `true` in test environments. * `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath @@ -316,9 +313,9 @@ Configure the network that will be enabled for the cluster and services. * `enabled`: Whether to compress the data in transit across the wire. The default is false. Requires Ceph Quincy (v17) or newer. Also see the kernel requirements above for encryption. - -> **NOTE:** Changing networking configuration after a Ceph cluster has been deployed is NOT -> supported and will result in a non-functioning cluster. +!!! caution + Changing networking configuration after a Ceph cluster has been deployed is NOT + supported and will result in a non-functioning cluster. #### Host Networking @@ -333,34 +330,40 @@ The selector keys are required to be `public` and `cluster` where each represent * `public`: client communications with the cluster (reads/writes) * `cluster`: internal Ceph replication network -If you want to learn more, please read +If you want to learn more, please read: + * [Ceph Networking reference](https://docs.ceph.com/docs/master/rados/configuration/network-config-ref/). * [Multus documentation](https://intel.github.io/multus-cni/doc/how-to-use.html) Based on the configuration, the operator will do the following: - 1. If only the `public` selector is specified, all communication will happen on that network -```yaml - network: - provider: multus - selectors: - public: rook-ceph/rook-public-nw -``` - 2. If only the `cluster` selector is specified, the internal cluster traffic* will happen on that network. All other traffic to mons, OSDs, and other daemons will be on the default network. -```yaml - network: - provider: multus - selectors: - cluster: rook-ceph/rook-cluster-nw -``` - 3. If both `public` and `cluster` selectors are specified the first one will run all the communication network and the second the internal cluster network* -```yaml - network: - provider: multus - selectors: - public: rook-ceph/rook-public-nw - cluster: rook-ceph/rook-cluster-nw -``` +1. If only the `public` selector is specified, all communication will happen on that network + + ```yaml + network: + provider: multus + selectors: + public: rook-ceph/rook-public-nw + ``` + +2. If only the `cluster` selector is specified, the internal cluster traffic* will happen on that network. All other traffic to mons, OSDs, and other daemons will be on the default network. + + ```yaml + network: + provider: multus + selectors: + cluster: rook-ceph/rook-cluster-nw + ``` + +3. If both `public` and `cluster` selectors are specified the first one will run all the communication network and the second the internal cluster network* + + ```yaml + network: + provider: multus + selectors: + public: rook-ceph/rook-public-nw + cluster: rook-ceph/rook-cluster-nw + ``` \* Internal cluster traffic includes OSD heartbeats, data replication, and data recovery @@ -399,10 +402,12 @@ spec: * Ipam type `whereabouts` is required because it makes sure that all the pods get a unique IP address from the multus network. * The NetworkAttachmentDefinition should be referenced along with the namespace in which it is present like `public: /`. e.g., the network attachment definition are in `default` namespace: + ```yaml public: default/rook-public-nw cluster: default/rook-cluster-nw ``` + * This format is required in order to use the NetworkAttachmentDefinition across namespaces. * In Openshift, to use a NetworkAttachmentDefinition (NAD) across namespaces, the NAD must be deployed in the `default` namespace. The NAD is then referenced with the namespace: `default/rook-public-nw` @@ -475,7 +480,7 @@ Below are the settings for host-based cluster. This type of cluster can specify * `config`: Device-specific config settings. See the [config settings](#osd-configuration-settings) below Host-based cluster only supports raw device and partition. Be sure to see the -[Ceph quickstart doc prerequisites](quickstart.md#prerequisites) for additional considerations. +[quickstart doc prerequisites](../Getting-Started/quickstart.md#prerequisites) for additional considerations. Below are the settings for a PVC-based cluster. @@ -510,7 +515,7 @@ The following are the settings for Storage Class Device Sets which can be config * `tuneFastDeviceClass`: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true.. * `volumeClaimTemplates`: A list of PVC templates to use for provisioning the underlying storage devices. * `resources.requests.storage`: The desired capacity for the underlying storage devices. - * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. This StorageClass should provide a raw block device, multipath device, or logical volume. Other types are not supported. If you want to use logical volume, please see [known issue of OSD on LV-backed PVC](ceph-common-issues.md#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc) + * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. This StorageClass should provide a raw block device, multipath device, or logical volume. Other types are not supported. If you want to use logical volume, please see [known issue of OSD on LV-backed PVC](../Troubleshooting/ceph-common-issues.md#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc) * `volumeMode`: The volume mode to be set for the PVC. Which should be Block * `accessModes`: The access mode for the PVC to be bound by OSD. * `schedulerName`: Scheduler name for OSD pod placement. (Optional) @@ -523,7 +528,7 @@ The following storage selection settings are specific to Ceph and do not apply t * `metadataDevice`: Name of a device to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a `metadataDevice` but that device is not used as a metadata device by Ceph. Notably, `ceph-volume` will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure. * `databaseSizeMB`: The size in MB of a bluestore database. Include quotes around the size. * `walSizeMB`: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size. -* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](ceph-pool-crd.md#spec). +* `deviceClass`: The [CRUSH device class](https://ceph.io/community/new-luminous-crush-device-classes/) to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either `hdd`, `ssd`, or `nvme` based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of [the pool spec's `deviceClass` field](Block-Storage/ceph-block-pool-crd.md#spec). * `initialWeight`: The initial OSD weight in TiB units. By default, this value is derived from OSD's capacity. * `primaryAffinity`: The [primary-affinity](https://docs.ceph.com/en/latest/rados/operations/crush-map/#primary-affinity) value of an OSD, within range `[0, 1]` (default: `1`). * `osdsPerDevice`**: The number of OSDs to create on each device. High performance devices such as NVMe can handle running multiple OSDs. If desired, this can be overridden for each node and each device. @@ -556,7 +561,6 @@ In stretch clusters, if the `arbiter` placement is specified, that placement wil Neither will the `arbiter` placement be merged with the `all` placement to allow the arbiter to be fully independent of other daemon placement. The remaining mons will still use the `mon` and/or `all` sections. - **NOTE:** Placement of OSD pods is controlled using the [Storage Class Device Set](#storage-class-device-sets), not the general `placement` configuration. A Placement configuration is specified (according to the kubernetes PodSpec) as: @@ -604,7 +608,8 @@ If a user configures a limit or request value that is too low, Rook will still r * `crashcollector`: 60MB * `mgr-sidecar`: 100MB limit, 40MB requests -> **HINT** The resources for MDS daemons are not configured in the Cluster. Refer to the [Ceph Filesystem CRD](ceph-filesystem-crd.md) instead. +!!! hint + The resources for MDS daemons are not configured in the Cluster. Refer to the [Ceph Filesystem CRD](Shared-Filesystem/ceph-filesystem-crd.md) instead. ### Resource Requirements/Limits @@ -706,12 +711,12 @@ Changing the liveness probe is an advanced operation and should rarely be necess The operator is regularly configuring and checking the health of the cluster. The results of the configuration and health checks can be seen in the `status` section of the CephCluster CR. -``` +```console kubectl -n rook-ceph get CephCluster -o yaml ``` ```yaml - ... +[...] status: ceph: health: HEALTH_OK @@ -755,28 +760,30 @@ The available space will be less that you may expect due to overhead in the OSDs ### Conditions The `conditions` represent the status of the Rook operator. -- If the cluster is fully configured and the operator is stable, the + +* If the cluster is fully configured and the operator is stable, the `Ready` condition is raised with `ClusterCreated` reason and no other conditions. The cluster will remain in the `Ready` condition after the first successful configuration since it is expected the storage is consumable from this point on. If there are issues preventing the storage layer from working, they are expected to show as Ceph health errors. -- If the cluster is externally connected successfully, the `Ready` condition will have the reason `ClusterConnected`. -- If the operator is currently being configured or the operator is checking for update, +* If the cluster is externally connected successfully, the `Ready` condition will have the reason `ClusterConnected`. +* If the operator is currently being configured or the operator is checking for update, there will be a `Progressing` condition. -- If there was a failure, the condition(s) status will be `false` and the `message` will +* If there was a failure, the condition(s) status will be `false` and the `message` will give a summary of the error. See the operator log for more details. ### Other Status There are several other properties for the overall status including: -- `message`, `phase`, and `state`: A summary of the overall current state of the cluster, which + +* `message`, `phase`, and `state`: A summary of the overall current state of the cluster, which is somewhat duplicated from the conditions for backward compatibility. -- `storage.deviceClasses`: The names of the types of storage devices that Ceph discovered +* `storage.deviceClasses`: The names of the types of storage devices that Ceph discovered in the cluster. These types will be `ssd` or `hdd` unless they have been overridden with the `crushDeviceClass` in the `storageClassDeviceSets`. -- `version`: The version of the Ceph image currently deployed. +* `version`: The version of the Ceph image currently deployed. -## Samples +## Examples Here are several samples for configuring Ceph clusters. Each of the samples must also include the namespace and corresponding access granted for management by the Ceph operator. See the [common cluster resources](#common-cluster-resources) below. @@ -899,7 +906,8 @@ spec: To control how many resources the Rook components can request/use, you can set requests and limits in Kubernetes for them. You can override these requests/limits for OSDs per node when using `useAllNodes: false` in the `node` item in the `nodes` list. -> **WARNING**: Before setting resource requests/limits, please take a look at the Ceph documentation for recommendations for each component: [Ceph - Hardware Recommendations](http://docs.ceph.com/docs/master/start/hardware-recommendations/). +!!! warning + Before setting resource requests/limits, please take a look at the Ceph documentation for recommendations for each component: [Ceph - Hardware Recommendations](http://docs.ceph.com/docs/master/start/hardware-recommendations/). ```yaml apiVersion: ceph.rook.io/v1 @@ -962,28 +970,26 @@ kubectl label node mynode topology.rook.io/rack=zone1-rack1 These labels would result in the following hierarchy for OSDs on that node (this command can be run in the Rook toolbox): ```console -ceph osd tree +$ ceph osd tree +ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF +-1 0.01358 root default +-5 0.01358 zone zone1 +-4 0.01358 rack rack1 +-3 0.01358 host mynode +0 hdd 0.00679 osd.0 up 1.00000 1.00000 +1 hdd 0.00679 osd.1 up 1.00000 1.00000 ``` ->``` ->ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF ->-1 0.01358 root default ->-5 0.01358 zone zone1 ->-4 0.01358 rack rack1 ->-3 0.01358 host mynode ->0 hdd 0.00679 osd.0 up 1.00000 1.00000 ->1 hdd 0.00679 osd.1 up 1.00000 1.00000 ->``` - Ceph requires unique names at every level in the hierarchy (CRUSH map). For example, you cannot have two racks with the same name that are in different zones. Racks in different zones must be named uniquely. Note that the `host` is added automatically to the hierarchy by Rook. The host cannot be specified with a topology label. All topology labels are optional. -> **HINT** When setting the node labels prior to `CephCluster` creation, these settings take immediate effect. However, applying this to an already deployed `CephCluster` requires removing each node from the cluster first and then re-adding it with new configuration to take effect. Do this node by node to keep your data safe! Check the result with `ceph osd tree` from the [Rook Toolbox](ceph-toolbox.md). The OSD tree should display the hierarchy for the nodes that already have been re-added. +!!! hint + When setting the node labels prior to `CephCluster` creation, these settings take immediate effect. However, applying this to an already deployed `CephCluster` requires removing each node from the cluster first and then re-adding it with new configuration to take effect. Do this node by node to keep your data safe! Check the result with `ceph osd tree` from the [Rook Toolbox](../Troubleshooting/ceph-toolbox.md). The OSD tree should display the hierarchy for the nodes that already have been re-added. -To utilize the `failureDomain` based on the node labels, specify the corresponding option in the [CephBlockPool](ceph-pool-crd.md) +To utilize the `failureDomain` based on the node labels, specify the corresponding option in the [CephBlockPool](Block-Storage/ceph-block-pool-crd.md) ```yaml apiVersion: ceph.rook.io/v1 @@ -1149,7 +1155,8 @@ So just taking the `storage` section this will give something like: - ReadWriteOnce ``` -> **NOTE**: Note that Rook only supports three naming convention for a given template: +!!! note + Note that Rook only supports three naming convention for a given template: * "data": represents the main OSD block device, where your data is being stored. * "metadata": represents the metadata (including block.db and block.wal) device used to store the Ceph Bluestore database for an OSD. @@ -1317,9 +1324,9 @@ The features available from the external cluster will vary depending on the vers ### External Cluster configuration -- Source cluster: The cluster providing the data, usually configured by [cephadm](https://docs.ceph.com/en/pacific/cephadm/#cephadm) +* Source cluster: The cluster providing the data, usually configured by [cephadm](https://docs.ceph.com/en/pacific/cephadm/#cephadm) -- Consumer cluster: The K8s cluster that will be consuming the external source cluster +* Consumer cluster: The K8s cluster that will be consuming the external source cluster ### Commands on the source Ceph cluster @@ -1327,20 +1334,21 @@ In order to configure an external Ceph cluster with Rook, we need to extract som 1. Run the python script [create-external-cluster-resources.py](/deploy/examples/create-external-cluster-resources.py) for creating all users and keys. -```console -python3 create-external-cluster-resources.py --rbd-data-pool-name --cephfs-filesystem-name --rgw-endpoint --namespace --rgw-pool-prefix --format bash -``` + ```console + python3 create-external-cluster-resources.py --rbd-data-pool-name --cephfs-filesystem-name --rgw-endpoint --namespace --rgw-pool-prefix --format bash + ``` -- `--namespace`: Namespace where CephCluster will run in the consumer cluster, for example `rook-ceph-external` -- `--format bash`: The format of the output -- `--rbd-data-pool-name`: The name of the RBD data pool -- `--cephfs-filesystem-name`: (optional) The name of the filesystem -- `--rgw-endpoint`: (optional) The RADOS Gateway endpoint in the format `:` -- `--rgw-pool-prefix`: (optional) The prefix of the RGW pools. If not specified, the default prefix is `default`. + * `--namespace`: Namespace where CephCluster will run in the consumer cluster, for example `rook-ceph-external` + * `--format bash`: The format of the output + * `--rbd-data-pool-name`: The name of the RBD data pool + * `--cephfs-filesystem-name`: (optional) The name of the filesystem + * `--rgw-endpoint`: (optional) The RADOS Gateway endpoint in the format `:` + * `--rgw-pool-prefix`: (optional) The prefix of the RGW pools. If not specified, the default prefix is `default`. 2. Copy the bash output. Example Output: + ```console export ROOK_EXTERNAL_FSID=797f411a-aafe-11ec-a254-fa163e1539f5 export ROOK_EXTERNAL_USERNAME=client.healthchecker @@ -1362,20 +1370,20 @@ export RGW_POOL_PREFIX=default 2. Paste the above output from `create-external-cluster-resources.py` into your current shell to allow importing the source data. 3. Run the [import](/deploy/examples/import-external-cluster.sh) script. -```console -. import-external-cluster.sh -``` + + ```console + . import-external-cluster.sh + ``` 4. Create [common-external.yaml](/deploy/examples/common-external.yaml) and [cluster-external.yaml](/deploy/examples/cluster-external.yaml) 5. Verify the consumer cluster is connected to the source ceph cluster: -```console -kubectl -n rook-ceph-external get CephCluster -``` ->``` ->NAME DATADIRHOSTPATH MONCOUNT AGE STATE HEALTH ->rook-ceph-external /var/lib/rook 162m Connected HEALTH_OK ->``` + + ```console + $ kubectl -n rook-ceph-external get CephCluster + NAME DATADIRHOSTPATH MONCOUNT AGE STATE HEALTH + rook-ceph-external /var/lib/rook 162m Connected HEALTH_OK + ``` ### Create StorageClass @@ -1386,43 +1394,43 @@ In this example, the pool `replicated_2g` exists in the source cluster. cat << EOF | kubectl apply -f - ``` ->``` ->apiVersion: storage.k8s.io/v1 ->kind: StorageClass ->metadata: -> name: rook-ceph-block-ext -># Change "rook-ceph" provisioner prefix to match the operator namespace if needed ->provisioner: rook-ceph.rbd.csi.ceph.com ->parameters: -> # clusterID is the namespace where the rook cluster is running -> clusterID: rook-ceph-external -> # Ceph pool into which the RBD image shall be created -> pool: replicated_2g -> -> # RBD image format. Defaults to "2". -> imageFormat: "2" -> -> # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. -> imageFeatures: layering -> -> # The secrets contain Ceph admin credentials. -> csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner -> csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph-external -> csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner -> csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph-external -> csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node -> csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph-external -> -> # Specify the filesystem type of the volume. If not specified, csi-provisioner -> # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock -> # in hyperconverged settings where the volume is mounted on the same node as the osds. -> csi.storage.k8s.io/fstype: ext4 -> -># Delete the rbd volume when a PVC is deleted ->reclaimPolicy: Delete ->allowVolumeExpansion: true ->EOF ->``` +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-ceph-block-ext +# Change "rook-ceph" provisioner prefix to match the operator namespace if needed +provisioner: rook-ceph.rbd.csi.ceph.com +parameters: + # clusterID is the namespace where the rook cluster is running + clusterID: rook-ceph-external + # Ceph pool into which the RBD image shall be created + pool: replicated_2g + + # RBD image format. Defaults to "2". + imageFormat: "2" + + # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + imageFeatures: layering + + # The secrets contain Ceph admin credentials. + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph-external + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph-external + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph-external + + # Specify the filesystem type of the volume. If not specified, csi-provisioner + # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock + # in hyperconverged settings where the volume is mounted on the same node as the osds. + csi.storage.k8s.io/fstype: ext4 + +# Delete the rbd volume when a PVC is deleted +reclaimPolicy: Delete +allowVolumeExpansion: true +EOF +``` You can now create a persistent volume based on this StorageClass. @@ -1454,6 +1462,7 @@ During deletion of a CephCluster resource, Rook protects against accidental or p of user data by blocking deletion if there are any other Rook-Ceph Custom Resources that reference the CephCluster being deleted. Rook will warn about which other resources are blocking deletion in three ways until all blocking resources are deleted: + 1. An event will be registered on the CephCluster resource 1. A status condition will be added to the CephCluster resource 1. An error will be added to the Rook-Ceph Operator log @@ -1466,7 +1475,7 @@ The `cleanupPolicy` has several fields: * `confirmation`: Only an empty string and `yes-really-destroy-data` are valid values for this field. If this setting is empty, the cleanupPolicy settings will be ignored and Rook will not cleanup any resources during cluster removal. - To reinstall the cluster, the admin would then be required to follow the [cleanup guide](ceph-teardown.md) to delete the data on hosts. + To reinstall the cluster, the admin would then be required to follow the [cleanup guide](../Storage-Configuration/ceph-teardown.md) to delete the data on hosts. If this setting is `yes-really-destroy-data`, the operator will automatically delete the data on hosts. Because this cleanup policy is destructive, after the confirmation is set to `yes-really-destroy-data` Rook will stop configuring the cluster as if the cluster is about to be destroyed. diff --git a/Documentation/ceph-nfs-crd.md b/Documentation/CRDs/ceph-nfs-crd.md similarity index 87% rename from Documentation/ceph-nfs-crd.md rename to Documentation/CRDs/ceph-nfs-crd.md index bb4435e209a8..ccb49905234d 100644 --- a/Documentation/ceph-nfs-crd.md +++ b/Documentation/CRDs/ceph-nfs-crd.md @@ -1,25 +1,22 @@ --- -title: NFS CRD -weight: 3100 -indent: true +title: NFS Server CRD --- -{% include_relative branch.liquid %} - -# Ceph NFS Server CRD ## Overview + Rook allows exporting NFS shares of a CephFilesystem or CephObjectStore through the CephNFS custom resource definition. This will spin up a cluster of [NFS Ganesha](https://github.com/nfs-ganesha/nfs-ganesha) servers that coordinate with one another via shared RADOS objects. The servers will be configured for NFSv4.1+ access only, as serving earlier protocols can inhibit responsiveness after a server restart. -> **WARNING**: Due to a number of Ceph issues and changes, Rook officially only supports Ceph -> v16.2.7 or higher for CephNFS. If you are using an earlier version, upgrade your Ceph version -> following the advice given in Rook's -> [v1.8 NFS docs](https://rook.github.io/docs/rook/v1.8/ceph-nfs-crd.html). +!!! warning + Due to a number of Ceph issues and changes, Rook officially only supports Ceph + v16.2.7 or higher for CephNFS. If you are using an earlier version, upgrade your Ceph version + following the advice given in Rook's + [v1.9 NFS docs](https://rook.github.io/docs/rook/v1.9/ceph-nfs-crd.html). -## Samples +## Examples ```yaml apiVersion: ceph.rook.io/v1 @@ -70,6 +67,7 @@ spec: ## NFS Settings ### Server + The `server` spec sets configuration for Rook-created NFS-Ganesha servers. * `active`: The number of active NFS servers. Rook supports creating more than one active NFS @@ -77,7 +75,7 @@ The `server` spec sets configuration for Rook-created NFS-Ganesha servers. [known issue](#serveractive-count-greater-than-1) below. * `placement`: Kubernetes placement restrictions to apply to NFS server Pod(s). This is similar to placement defined for daemons configured by the - [CephCluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml). + [CephCluster CRD](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml). * `annotations`: Kubernetes annotations to apply to NFS server Pod(s) * `labels`: Kubernetes labels to apply to NFS server Pod(s) * `resources`: Kubernetes resource requests and limits to set on NFS server Pod(s) @@ -86,30 +84,34 @@ The `server` spec sets configuration for Rook-created NFS-Ganesha servers. Default value: NIV_INFO
Supported values: NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG | NIV_FULL_DEBUG | NB_LOG_LEVEL - ## Creating Exports + When a CephNFS is first created, all NFS daemons within the CephNFS cluster will share a configuration with no exports defined. ### Using the Ceph Dashboard + Exports can be created via the [Ceph dashboard](https://docs.ceph.com/en/latest/mgr/dashboard/#nfs-ganesha-management) for Ceph v16 -as well. To enable and use the Ceph dashboard in Rook, see [here](ceph-dashboard.md). +as well. To enable and use the Ceph dashboard in Rook, see [here](../Storage-Configuration/Monitoring/ceph-dashboard.md). ### Using the Ceph CLI + The Ceph CLI can be used from the Rook toolbox pod to create and manage NFS exports. To do so, first ensure the necessary Ceph mgr modules are enabled, if necessary, and that the Ceph orchestrator backend is set to Rook. -> #### **Enable the Ceph orchestrator if necessary** -> * Required for Ceph v16.2.7 and below -> * Optional for Ceph v16.2.8 and above -> * Must be disabled for Ceph v17.2.0 due to a [Ceph regression](#ceph-v1720) -> ```console -> ceph mgr module enable rook -> ceph mgr module enable nfs -> ceph orch set backend rook -> ``` +#### **Enable the Ceph orchestrator if necessary** + +* Required for Ceph v16.2.7 and below +* Optional for Ceph v16.2.8 and above +* Must be disabled for Ceph v17.2.0 due to a [Ceph regression](#ceph-v1720) + +```console +ceph mgr module enable rook +ceph mgr module enable nfs +ceph orch set backend rook +``` [Ceph's NFS CLI](https://docs.ceph.com/en/latest/mgr/nfs/#export-management) can create NFS exports that are backed by [CephFS](https://docs.ceph.com/en/latest/cephfs/nfs/) (a CephFilesystem) or @@ -119,16 +121,16 @@ which is the CephNFS name in the Rook context. For creating an NFS export for the CephNFS and CephFilesystem example manifests, the below command can be used. This creates an export for the `/test` pseudo path. + ```console ceph nfs export create cephfs my-nfs /test myfs ``` The below command will list the current NFS exports for the example CephNFS cluster, which will give the output shown for the current example. + ```console -ceph nfs export ls my-nfs -``` -``` +$ ceph nfs export ls my-nfs [ "/test" ] @@ -136,10 +138,9 @@ ceph nfs export ls my-nfs The simple `/test` export's info can be listed as well. Notice from the example that only NFS protocol v4 via TCP is supported. + ```console -ceph nfs export info my-nfs /test -``` -``` +$ ceph nfs export info my-nfs /test { "export_id": 1, "path": "/", @@ -166,12 +167,14 @@ ceph nfs export info my-nfs /test If you are done managing NFS exports and don't need the Ceph orchestrator module enabled for anything else, it may be preferable to disable the Rook and NFS mgr modules to free up a small amount of RAM in the Ceph mgr Pod. + ```console ceph orch set backend "" ceph mgr module disable rook ``` ### Mounting exports + Each CephNFS server has a unique Kubernetes Service. This is because NFS clients can't readily handle NFS failover. CephNFS services are named with the pattern `rook-ceph-nfs--` `` is a unique letter ID (e.g., a, b, c, etc.) for a given @@ -180,19 +183,20 @@ NFS server. For example, `rook-ceph-nfs-my-nfs-a`. For each NFS client, choose an NFS service to use for the connection. With NFS v4, you can mount an export by its path using a mount command like below. You can mount all exports at once by omitting the export path and leaving the directory as just `/`. -``` + +```console mount -t nfs4 -o proto=tcp :/ ``` - ## Exposing the NFS server outside of the Kubernetes cluster + Use a LoadBalancer Service to expose an NFS server (and its exports) outside of the Kubernetes cluster. The Service's endpoint can be used as the NFS service address when [mounting the export manually](#mounting-exports). We provide an example Service here: -[`deploy/examples/nfs-load-balancer.yaml`](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples). - +[`deploy/examples/nfs-load-balancer.yaml`](https://github.com/rook/rook/tree/master/deploy/examples). ## Scaling the active server count + It is possible to scale the size of the cluster up or down by modifying the `spec.server.active` field. Scaling the cluster size up can be done at will. Once the new server comes up, clients can be assigned to it immediately. @@ -202,12 +206,14 @@ started. Scaling down the cluster requires that clients be migrated from servers eliminated to others. That process is currently a manual one and should be performed before reducing the size of the cluster. -> **WARNING:** see the [known issue](#serveractive-count-greater-than-1) below about setting this -> value greater than one. - +!!! warning + See the [known issue](#serveractive-count-greater-than-1) below about setting this + value greater than one. ## Known issues + ### server.active count greater than 1 + * Active-active scale out does not work well with the NFS protocol. If one NFS server in a cluster is offline, other servers may block client requests until the offline server returns, which may not always happen due to the Kubernetes scheduler. @@ -215,30 +221,34 @@ the size of the cluster. benefits your use case. ### Ceph v17.2.0 + * Ceph NFS management with the Rook mgr module enabled has a breaking regression with the Ceph Quincy v17.2.0 release. * Workaround: Leave Ceph's Rook orchestrator mgr module disabled. If you have enabled it, you must disable it using the snippet below from the toolbox. + ```console ceph orch set backend "" ceph mgr module disable rook ``` - ## Advanced configuration + All CephNFS daemons are configured using shared RADOS objects stored in a Ceph pool named `.nfs`. Users can modify the configuration object for each CephNFS cluster if they wish to customize the configuration. ### Changing configuration of the .nfs pool + By default, Rook creates the `.nfs` pool with Ceph's default configuration. If you wish to change the configuration of this pool (for example to change its failure domain or replication factor), you can create a CephBlockPool with the `spec.name` field set to `.nfs`. This pool **must** be replicated and **cannot** be erasure coded. -[`deploy/examples/nfs.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/nfs.yaml) +[`deploy/examples/nfs.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/nfs.yaml) contains a sample for reference. ### Adding custom NFS-Ganesha config file changes + The NFS-Ganesha config file format for these objects is documented in the [NFS-Ganesha project](https://github.com/nfs-ganesha/nfs-ganesha/wiki). @@ -249,9 +259,7 @@ It is best not to modify any of the export objects created by Ceph so as not to Ceph's export management. ```console -rados --pool --namespace get conf-nfs. - -``` -``` +$ rados --pool --namespace get conf-nfs. - %url "rados:////export-1" %url "rados:////export-2" ``` @@ -262,16 +270,17 @@ configuration objects. Of note, it is possible to pre-populate the NFS configuration and export objects prior to creating CephNFS server clusters. - ## Ceph CSI NFS provisioner and NFS CSI driver -> **EXPERIMENTAL**: this feature is experimental, and we do not guarantee it is bug-free, nor will -> we support upgrades to future versions + +!!! attention + This feature is experimental, and we do not guarantee it is bug-free, nor will + we support upgrades to future versions In version 1.9.1, Rook is able to deploy the experimental NFS Ceph CSI driver. This requires Ceph CSI version 3.6.0 or above. We recommend Ceph v16.2.7 or above. For this section, we will refer to Rook's deployment examples in the -[deploy/examples](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples) directory. +[deploy/examples](https://github.com/rook/rook/tree/master/deploy/examples) directory. The Ceph CSI NFS provisioner and driver require additional RBAC to operate. Apply the `deploy/examples/csi/nfs/rbac.yaml` manifest to deploy the additional resources. @@ -279,12 +288,14 @@ The Ceph CSI NFS provisioner and driver require additional RBAC to operate. Appl Rook will only deploy the Ceph CSI NFS provisioner and driver components when the `ROOK_CSI_ENABLE_NFS` config is set to `"true"` in the `rook-ceph-operator-config` configmap. Change the value in your manifest, or patch the resource as below. -```sh + +```console kubectl --namespace rook-ceph patch configmap rook-ceph-operator-config --type merge --patch '{"data":{"ROOK_CSI_ENABLE_NFS": "true"}}' ``` -> **NOTE:** The rook-ceph operator Helm chart will deploy the required RBAC and enable the driver -> components if `csi.nfs.enabled` is set to `true`. +!!! note + The rook-ceph operator Helm chart will deploy the required RBAC and enable the driver + components if `csi.nfs.enabled` is set to `true`. In order to create NFS exports via the CSI driver, you must first create a CephFilesystem to serve as the underlying storage for the exports, and you must create a CephNFS to run an NFS server that diff --git a/Documentation/Contributing/.pages b/Documentation/Contributing/.pages new file mode 100644 index 000000000000..e98330f8886a --- /dev/null +++ b/Documentation/Contributing/.pages @@ -0,0 +1,6 @@ +nav: + - development-flow.md + - development-environment.md + - documentation.md + - storage-providers.md + - ... diff --git a/Documentation/development-environment.md b/Documentation/Contributing/development-environment.md similarity index 85% rename from Documentation/development-environment.md rename to Documentation/Contributing/development-environment.md index c1f7bc472dd7..297a9ecfa863 100644 --- a/Documentation/development-environment.md +++ b/Documentation/Contributing/development-environment.md @@ -1,15 +1,13 @@ --- title: Developer Environment -weight: 12030 -indent: true --- -# Install Kubernetes +## Install Kubernetes You can choose any Kubernetes install of your choice. The test framework only depends on `kubectl` being configured. To install `kubectl`, please see the [official guide](https://kubernetes.io/docs/tasks/tools/#kubectl). -# Minikube +## Minikube The developers of Rook are working on Minikube and thus it is the recommended way to quickly get Rook up and running. Minikube should not be used for production but the Rook authors @@ -51,8 +49,10 @@ Use [helm.sh](/tests/scripts/helm.sh) to install Helm and set up Rook charts def - To install and set up Helm charts for Rook run `tests/scripts/helm.sh up`. - To clean up `tests/scripts/helm.sh clean`. -**NOTE:** These helper scripts depend on some artifacts under the `_output/` directory generated during build time. -These scripts should be run from the project root. +!!! note + These helper scripts depend on some artifacts under the `_output/` directory generated during build time. + These scripts should be run from the project root. -**NOTE**: If Helm is not available in your `PATH`, Helm will be downloaded to a temporary directory +!!! note + If Helm is not available in your `PATH`, Helm will be downloaded to a temporary directory (`/tmp/rook-tests-scripts-helm`) and used from that directory. diff --git a/Documentation/development-flow.md b/Documentation/Contributing/development-flow.md similarity index 99% rename from Documentation/development-flow.md rename to Documentation/Contributing/development-flow.md index 29ec2caf2070..4919404a5905 100644 --- a/Documentation/development-flow.md +++ b/Documentation/Contributing/development-flow.md @@ -1,10 +1,7 @@ --- -title: Contributing -weight: 12000 +title: Development Flow --- -# Contributing - Thank you for your time and effort to help us improve Rook! Here are a few steps to get started. If you have any questions, don't hesitate to reach out to us on our [Slack](https://Rook-io.slack.com) dev channel. @@ -246,6 +243,7 @@ for every possible input. Higher-level code units that are built from smaller, a can more easily verify that the units are combined together correctly. Common cases that may need tests: + * the feature is enabled * the feature is disabled * the feature is only partially enabled, for every possible way it can be partially enabled @@ -257,7 +255,6 @@ Common cases that may need tests: * an input is specified incorrectly, for each input * a resource the code relies on doesn't exist, for each dependency - #### Running the Integration Tests Rook's upstream continuous integration (CI) tests will run integration tests against your changes diff --git a/Documentation/Contributing/documentation.md b/Documentation/Contributing/documentation.md new file mode 100644 index 000000000000..7e981e4b7342 --- /dev/null +++ b/Documentation/Contributing/documentation.md @@ -0,0 +1,17 @@ +--- +title: Documentation +--- + +We are using [MkDocs](https://www.mkdocs.org/) with the [Material for MkDocs theme](https://squidfunk.github.io/mkdocs-material/). + +## Markdown Extensions + +Thanks to the MkDocs theme we have certain "markdown syntax extensions" available: + +* Admonitions +* Footnotes +* Icons, Emojis +* Lists (Task lists) +* And some more + +For a whole list of features [Reference - Material for MkDocs](https://squidfunk.github.io/mkdocs-material/reference/). diff --git a/Documentation/storage-providers.md b/Documentation/Contributing/storage-providers.md similarity index 99% rename from Documentation/storage-providers.md rename to Documentation/Contributing/storage-providers.md index 052a15f19fd8..40894aa1ed84 100644 --- a/Documentation/storage-providers.md +++ b/Documentation/Contributing/storage-providers.md @@ -1,11 +1,7 @@ --- title: Storage Providers -weight: 12050 -indent: true --- -# Storage Providers - Rook is the home for operators for multiple storage providers. Each of these storage providers has specific requirements and each of them is very independent. There is no runtime dependency between the storage providers. Development is where the storage providers benefit from one another. diff --git a/Documentation/Getting-Started/.pages b/Documentation/Getting-Started/.pages new file mode 100644 index 000000000000..d67756b7c6f1 --- /dev/null +++ b/Documentation/Getting-Started/.pages @@ -0,0 +1,7 @@ +nav: + - intro.md + - Prerequisites + - quickstart.md + - storage-architecture.md + - example-configurations.md + - ... diff --git a/Documentation/Getting-Started/Prerequisites/.pages b/Documentation/Getting-Started/Prerequisites/.pages new file mode 100644 index 000000000000..00a3821d2efb --- /dev/null +++ b/Documentation/Getting-Started/Prerequisites/.pages @@ -0,0 +1,3 @@ +nav: + - prerequisites.md + - ... diff --git a/Documentation/authenticated-registry.md b/Documentation/Getting-Started/Prerequisites/authenticated-registry.md similarity index 76% rename from Documentation/authenticated-registry.md rename to Documentation/Getting-Started/Prerequisites/authenticated-registry.md index e359352b2c3d..f8265cbdb032 100644 --- a/Documentation/authenticated-registry.md +++ b/Documentation/Getting-Started/Prerequisites/authenticated-registry.md @@ -1,11 +1,7 @@ --- -title: Authenticated Registries -weight: 1100 -indent: true +title: Authenticated Container Registries --- -## Authenticated docker registries - If you want to use an image from authenticated docker registry (e.g. for image cache/mirror), you'll need to add an `imagePullSecret` to all relevant service accounts. This way all pods created by the operator (for service account: `rook-ceph-system`) or all new pods in the namespace (for service account: `default`) will have the `imagePullSecret` added @@ -13,18 +9,14 @@ to their spec. The whole process is described in the [official kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#add-imagepullsecrets-to-a-service-account). -### Example setup for a ceph cluster +## Example setup for a ceph cluster To get you started, here's a quick rundown for the ceph example from the [quickstart guide](/Documentation/quickstart.md). -First, we'll create the secret for our registry as described [here](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod): +First, we'll create the secret for our registry as described [here](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) (the secret will be created in the `rook-ceph` namespace, make sure to change it if your Rook Ceph Operator/Cluster is in another namespace): ```console -# for namespace rook-ceph -$ kubectl -n rook-ceph create secret docker-registry my-registry-secret --docker-server=DOCKER_REGISTRY_SERVER --docker-username=DOCKER_USER --docker-password=DOCKER_PASSWORD --docker-email=DOCKER_EMAIL - -# and for namespace rook-ceph (cluster) -$ kubectl -n rook-ceph create secret docker-registry my-registry-secret --docker-server=DOCKER_REGISTRY_SERVER --docker-username=DOCKER_USER --docker-password=DOCKER_PASSWORD --docker-email=DOCKER_EMAIL +kubectl -n rook-ceph create secret docker-registry my-registry-secret --docker-server=DOCKER_REGISTRY_SERVER --docker-username=DOCKER_USER --docker-password=DOCKER_PASSWORD --docker-email=DOCKER_EMAIL ``` Next we'll add the following snippet to all relevant service accounts as described [here](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#add-imagepullsecrets-to-a-service-account): @@ -51,7 +43,7 @@ Since it's the same procedure for all service accounts, here is just one example kubectl -n rook-ceph edit serviceaccount default ``` -```yaml +```yaml hl_lines="9-10" apiVersion: v1 kind: ServiceAccount metadata: @@ -59,8 +51,9 @@ metadata: namespace: rook-ceph secrets: - name: default-token-12345 -imagePullSecrets: # here are the new -- name: my-registry-secret # parts +# Add the highlighted lines: +imagePullSecrets: +- name: my-registry-secret ``` After doing this for all service accounts all pods should be able to pull the image from your registry. diff --git a/Documentation/pod-security-policies.md b/Documentation/Getting-Started/Prerequisites/pod-security-policies.md similarity index 75% rename from Documentation/pod-security-policies.md rename to Documentation/Getting-Started/Prerequisites/pod-security-policies.md index 8feb1ba78f1b..d6e673368ca5 100644 --- a/Documentation/pod-security-policies.md +++ b/Documentation/Getting-Started/Prerequisites/pod-security-policies.md @@ -1,32 +1,27 @@ --- title: Pod Security Policies -weight: 1300 -indent: true --- -{% include_relative branch.liquid %} - -## Pod Security Policies Rook requires privileges to manage the storage in your cluster. If you have Pod Security Policies enabled please review this document. By default, Kubernetes clusters do not have PSPs enabled so you may be able to skip this document. If you are configuring Ceph on OpenShift, the Ceph walkthrough will configure the PSPs as well -when you start the operator with [operator-openshift.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/operator-openshift.yaml). +when you start the operator with [operator-openshift.yaml](https://github.com/rook/rook/blob/master/deploy/examples/operator-openshift.yaml). Creating the Rook operator requires privileges for setting up RBAC. To launch the operator you need to have created your user certificate that is bound to ClusterRole `cluster-admin`. -### RBAC for PodSecurityPolicies +## RBAC for PodSecurityPolicies If you have activated the [PodSecurityPolicy Admission Controller](https://kubernetes.io/docs/admin/admission-controllers/#podsecuritypolicy) and thus are using [PodSecurityPolicies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/), you will require additional `(Cluster)RoleBindings` for the different `ServiceAccounts` Rook uses to start the Rook Storage Pods. Security policies will differ for different backends. See Ceph's Pod Security Policies set up in -[common.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/common.yaml) +[common.yaml](https://github.com/rook/rook/blob/master/deploy/examples/common.yaml) for an example of how this is done in practice. -### PodSecurityPolicy +## PodSecurityPolicy You need at least one `PodSecurityPolicy` that allows privileged `Pod` execution. Here is an example which should be more permissive than is needed for any backend: @@ -55,9 +50,10 @@ spec: hostNetwork: false ``` -**Hint**: Allowing `hostNetwork` usage is required when using `hostNetwork: true` in a Cluster `CustomResourceDefinition`! -You are then also required to allow the usage of `hostPorts` in the `PodSecurityPolicy`. The given -port range will allow all ports: +!!! hint + Allowing `hostNetwork` usage is required when using `hostNetwork: true` in a Cluster `CustomResourceDefinition`! + You are then also required to allow the usage of `hostPorts` in the `PodSecurityPolicy`. The given + port range will allow all ports: ```yaml hostPorts: diff --git a/Documentation/pre-reqs.md b/Documentation/Getting-Started/Prerequisites/prerequisites.md similarity index 70% rename from Documentation/pre-reqs.md rename to Documentation/Getting-Started/Prerequisites/prerequisites.md index d8b0f998d8a9..a84f61c22c4d 100644 --- a/Documentation/pre-reqs.md +++ b/Documentation/Getting-Started/Prerequisites/prerequisites.md @@ -1,38 +1,33 @@ --- title: Prerequisites -weight: 1000 --- -{% include_relative branch.liquid %} - -# Prerequisites Rook can be installed on any existing Kubernetes cluster as long as it meets the minimum version and Rook is granted the required privileges (see below for more information). ## Minimum Version -Kubernetes **v1.16** or higher is supported for the Ceph operator. +Kubernetes **v1.17** or higher is supported for the Ceph operator. ## Ceph Prerequisites In order to configure the Ceph storage cluster, at least one of these local storage options are required: -- Raw devices (no partitions or formatted filesystems) -- Raw partitions (no formatted filesystem) -- PVs available from a storage class in `block` mode + +* Raw devices (no partitions or formatted filesystems) +* Raw partitions (no formatted filesystem) +* PVs available from a storage class in `block` mode You can confirm whether your partitions or devices are formatted with filesystems with the following command. ```console -lsblk -f +$ lsblk -f +NAME FSTYPE LABEL UUID MOUNTPOINT +vda +└─vda1 LVM2_member >eSO50t-GkUV-YKTH-WsGq-hNJY-eKNf-3i07IB + ├─ubuntu--vg-root ext4 c2366f76-6e21-4f10-a8f3-6776212e2fe4 / + └─ubuntu--vg-swap_1 swap 9492a3dc-ad75-47cd-9596-678e8cf17ff9 [SWAP] +vdb ``` ->``` ->NAME FSTYPE LABEL UUID MOUNTPOINT ->vda ->└─vda1 LVM2_member >eSO50t-GkUV-YKTH-WsGq-hNJY-eKNf-3i07IB -> ├─ubuntu--vg-root ext4 c2366f76-6e21-4f10-a8f3-6776212e2fe4 / -> └─ubuntu--vg-swap_1 swap 9492a3dc-ad75-47cd-9596-678e8cf17ff9 [SWAP] ->vdb ->``` If the `FSTYPE` field is not empty, there is a filesystem on top of the corresponding device. In this example, you can use `vdb` for Ceph and can't use `vda` or its partitions. @@ -49,38 +44,40 @@ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.7 ## LVM package Ceph OSDs have a dependency on LVM in the following scenarios: -- OSDs are created on raw devices or partitions -- If encryption is enabled (`encryptedDevice: true` in the cluster CR) -- A `metadata` device is specified + +* OSDs are created on raw devices or partitions +* If encryption is enabled (`encryptedDevice: true` in the cluster CR) +* A `metadata` device is specified LVM is not required for OSDs in these scenarios: -- Creating OSDs on PVCs using the `storageClassDeviceSets` + +* Creating OSDs on PVCs using the `storageClassDeviceSets` If LVM is required for your scenario, LVM needs to be available on the hosts where OSDs will be running. Some Linux distributions do not ship with the `lvm2` package. This package is required on all storage nodes in your k8s cluster to run Ceph OSDs. Without this package even though Rook will be able to successfully create the Ceph OSDs, when a node is rebooted the OSD pods running on the restarted node will **fail to start**. Please install LVM using your Linux distribution's package manager. For example: -CentOS: +**CentOS**: ```console sudo yum install -y lvm2 ``` -Ubuntu: +**Ubuntu**: ```console sudo apt-get install -y lvm2 ``` -RancherOS: +**RancherOS**: -- Since version [1.5.0](https://github.com/rancher/os/issues/2551) LVM is supported -- Logical volumes [will not be activated](https://github.com/rook/rook/issues/5027) during the boot process. You need to add an [runcmd command](https://rancher.com/docs/os/v1.x/en/installation/configuration/running-commands/) for that. +* Since version [1.5.0](https://github.com/rancher/os/issues/2551) LVM is supported +* Logical volumes [will not be activated](https://github.com/rook/rook/issues/5027) during the boot process. You need to add an [runcmd command](https://rancher.com/docs/os/v1.x/en/installation/configuration/running-commands/) for that. ```yaml runcmd: -- [ vgchange, -ay ] +- [ "vgchange", "-ay" ] ``` ## Kernel @@ -91,7 +88,7 @@ Ceph requires a Linux kernel built with the RBD module. Many Linux distributions For example, the GKE Container-Optimised OS (COS) does not have RBD. You can test your Kubernetes nodes by running `modprobe rbd`. -If it says 'not found', you may have to [rebuild your kernel](common-issues.md#rook-agent-rbd-module-missing-error) +If it says 'not found', you may have to rebuild your kernel and include at least the `rbd` module or choose a different Linux distribution. ### CephFS diff --git a/Documentation/ceph-openshift.md b/Documentation/Getting-Started/ceph-openshift.md similarity index 83% rename from Documentation/ceph-openshift.md rename to Documentation/Getting-Started/ceph-openshift.md index 378c7a3a23b3..28ea0296b92f 100644 --- a/Documentation/ceph-openshift.md +++ b/Documentation/Getting-Started/ceph-openshift.md @@ -1,18 +1,15 @@ --- title: OpenShift -weight: 2060 -indent: true --- -{% include_relative branch.liquid %} # OpenShift [OpenShift](https://www.openshift.com/) adds a number of security and other enhancements to Kubernetes. In particular, [security context constraints](https://blog.openshift.com/understanding-service-accounts-sccs/) allow the cluster admin to define exactly which permissions are allowed to pods running in the cluster. You will need to define those permissions that allow the Rook pods to run. -The settings for Rook in OpenShift are described below, and are also included in the [example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples): +The settings for Rook in OpenShift are described below, and are also included in the [example yaml files](https://github.com/rook/rook/blob/master/deploy/examples): -* `operator-openshift.yaml`: Creates the security context constraints and starts the operator deployment -* `object-openshift.yaml`: Creates an object store with rgw listening on a valid port number for OpenShift +* [`operator-openshift.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator-openshift.yaml): Creates the security context constraints and starts the operator deployment +* [`object-openshift.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/object-openshift.yaml): Creates an object store with rgw listening on a valid port number for OpenShift ## TL;DR @@ -37,7 +34,8 @@ To orchestrate the storage platform, Rook requires the following access in the c Before starting the Rook operator or cluster, create the security context constraints needed by the Rook pods. The following yaml is found in `operator-openshift.yaml` under `/deploy/examples`. -> **NOTE**: Older versions of OpenShift may require `apiVersion: v1`. +!!! hint + Older versions of OpenShift may require `apiVersion: v1`. Important to note is that if you plan on running Rook in namespaces other than the default `rook-ceph`, the example scc will need to be modified to accommodate for your namespaces where the Rook pods are running. @@ -78,7 +76,7 @@ The cluster settings in `cluster.yaml` are largely isolated from the differences ### Object Store Settings -In OpenShift, ports less than 1024 cannot be bound. In the [object store CRD](ceph-object.md), ensure the port is modified to meet this requirement. +In OpenShift, ports less than 1024 cannot be bound. In the [object store CRD](../Storage-Configuration/Object-Storage-RGW/object-storage.md), ensure the port is modified to meet this requirement. ```yaml gateway: diff --git a/Documentation/media/kubernetes.png b/Documentation/Getting-Started/ceph-storage/kubernetes.png similarity index 100% rename from Documentation/media/kubernetes.png rename to Documentation/Getting-Started/ceph-storage/kubernetes.png diff --git a/Documentation/Getting-Started/example-configurations.md b/Documentation/Getting-Started/example-configurations.md new file mode 100644 index 000000000000..f6e2ad87f6f3 --- /dev/null +++ b/Documentation/Getting-Started/example-configurations.md @@ -0,0 +1,104 @@ +--- +title: Example Configurations +--- + +Configuration for Rook and Ceph can be configured in multiple ways to provide block devices, shared filesystem volumes or object storage in a kubernetes namespace. We have provided several examples to simplify storage setup, but remember there are many tunables and you will need to decide what settings work for your use case and environment. + +See the **[example yaml files](https://github.com/rook/rook/blob/master/deploy/examples)** folder for all the rook/ceph setup example spec files. + +## Common Resources + +The first step to deploy Rook is to create the CRDs and other common resources. The configuration for these resources will be the same for most deployments. +The [crds.yaml](https://github.com/rook/rook/blob/master/deploy/examples/crds.yaml) and +[common.yaml](https://github.com/rook/rook/blob/master/deploy/examples/common.yaml) sets these resources up. + +```console +kubectl create -f crds.yaml -f common.yaml +``` + +The examples all assume the operator and all Ceph daemons will be started in the same namespace. If you want to deploy the operator in a separate namespace, see the comments throughout `common.yaml`. + +## Operator + +After the common resources are created, the next step is to create the Operator deployment. Several spec file examples are provided in [this directory](https://github.com/rook/rook/blob/master/deploy/examples/): + +* [`operator.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator.yaml): The most common settings for production deployments + * `kubectl create -f operator.yaml` +* [`operator-openshift.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator-openshift.yaml): Includes all of the operator settings for running a basic Rook cluster in an OpenShift environment. You will also want to review the [OpenShift Prerequisites](../Getting-Started/ceph-openshift.md) to confirm the settings. + * `oc create -f operator-openshift.yaml` + +Settings for the operator are configured through environment variables on the operator deployment. The individual settings are documented in [operator.yaml](https://github.com/rook/rook/blob/master/deploy/examples/operator.yaml). + +## Cluster CRD + +Now that your operator is running, let's create your Ceph storage cluster. This CR contains the most critical settings +that will influence how the operator configures the storage. It is important to understand the various ways to configure +the cluster. These examples represent a very small set of the different ways to configure the storage. + +* [`cluster.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml): This file contains common settings for a production storage cluster. Requires at least three worker nodes. +* [`cluster-test.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-test.yaml): Settings for a test cluster where redundancy is not configured. Requires only a single node. +* [`cluster-on-pvc.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-on-pvc.yaml): This file contains common settings for backing the Ceph Mons and OSDs by PVs. Useful when running in cloud environments or where local PVs have been created for Ceph to consume. +* [`cluster-external.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-external.yaml): Connect to an [external Ceph cluster](../CRDs/ceph-cluster-crd.md#external-cluster) with minimal access to monitor the health of the cluster and connect to the storage. +* [`cluster-external-management.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-external-management.yaml): Connect to an [external Ceph cluster](../CRDs/ceph-cluster-crd.md#external-cluster) with the admin key of the external cluster to enable + remote creation of pools and configure services such as an [Object Store](../Storage-Configuration/Object-Storage-RGW/object-storage.md) or a [Shared Filesystem](../Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md). +* [`cluster-stretched.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-stretched.yaml): Create a cluster in "stretched" mode, with five mons stretched across three zones, and the OSDs across two zones. See the [Stretch documentation](../CRDs/ceph-cluster-crd.md#stretch-cluster). + +See the [Cluster CRD](../CRDs/ceph-cluster-crd.md) topic for more details and more examples for the settings. + +## Setting up consumable storage + +Now we are ready to setup [block](https://ceph.com/ceph-storage/block-storage/), [shared filesystem](https://ceph.com/ceph-storage/file-system/) or [object storage](https://ceph.com/ceph-storage/object-storage/) in the Rook Ceph cluster. These kinds of storage are respectively referred to as CephBlockPool, CephFilesystem and CephObjectStore in the spec files. + +### Block Devices + +Ceph can provide raw block device volumes to pods. Each example below sets up a storage class which can then be used to provision a block device in kubernetes pods. The storage class is defined with [a pool](http://docs.ceph.com/docs/master/rados/operations/pools/) which defines the level of data redundancy in Ceph: + +* [`storageclass.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/storageclass.yaml): This example illustrates replication of 3 for production scenarios and requires at least three worker nodes. Your data is replicated on three different kubernetes worker nodes and intermittent or long-lasting single node failures will not result in data unavailability or loss. +* [`storageclass-ec.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/storageclass-ec.yaml): Configures erasure coding for data durability rather than replication. [Ceph's erasure coding](http://docs.ceph.com/docs/master/rados/operations/erasure-code/) is more efficient than replication so you can get high reliability without the 3x replication cost of the preceding example (but at the cost of higher computational encoding and decoding costs on the worker nodes). Erasure coding requires at least three worker nodes. See the [Erasure coding](../CRDs/Block-Storage/ceph-block-pool-crd.md#erasure-coded) documentation for more details. +* [`storageclass-test.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/storageclass-test.yaml): Replication of 1 for test scenarios and it requires only a single node. Do not use this for applications that store valuable data or have high-availability storage requirements, since a single node failure can result in data loss. + +The storage classes are found in different sub-directories depending on the driver: + +* `csi/rbd`: The CSI driver for block devices. This is the preferred driver going forward. + +See the [Ceph Pool CRD](../CRDs/Block-Storage/ceph-block-pool-crd.md) topic for more details on the settings. + +### Shared Filesystem + +Ceph filesystem (CephFS) allows the user to 'mount' a shared posix-compliant folder into one or more hosts (pods in the container world). This storage is similar to NFS shared storage or CIFS shared folders, as explained [here](https://ceph.com/ceph-storage/file-system/). + +File storage contains multiple pools that can be configured for different scenarios: + +* [`filesystem.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/filesystem.yaml): Replication of 3 for production scenarios. Requires at least three worker nodes. +* [`filesystem-ec.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/filesystem-ec.yaml): Erasure coding for production scenarios. Requires at least three worker nodes. +* [`filesystem-test.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/filesystem-test.yaml): Replication of 1 for test scenarios. Requires only a single node. + +Dynamic provisioning is possible with the CSI driver. The storage class for shared filesystems is found in the [`csi/cephfs`](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs) directory. + +See the [Shared Filesystem CRD](../CRDs/Shared-Filesystem/ceph-filesystem-crd.md) topic for more details on the settings. + +### Object Storage + +Ceph supports storing blobs of data called objects that support HTTP(s)-type get/put/post and delete semantics. This storage is similar to AWS S3 storage, for example. + +Object storage contains multiple pools that can be configured for different scenarios: + +* [`object.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/object.yaml): Replication of 3 for production scenarios. Requires at least three worker nodes. +* [`object-openshift.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/object-openshift.yaml): Replication of 3 with rgw in a port range valid for OpenShift. Requires at least three worker nodes. +* [`object-ec.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/object-ec.yaml): Erasure coding rather than replication for production scenarios. Requires at least three worker nodes. +* [`object-test.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/object-test.yaml): Replication of 1 for test scenarios. Requires only a single node. + +See the [Object Store CRD](../CRDs/Object-Storage/ceph-object-store-crd.md) topic for more details on the settings. + +### Object Storage User + +* [`object-user.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/object-user.yaml): Creates a simple object storage user and generates credentials for the S3 API + +### Object Storage Buckets + +The Ceph operator also runs an object store bucket provisioner which can grant access to existing buckets or dynamically provision new buckets. + +* [object-bucket-claim-retain.yaml](https://github.com/rook/rook/blob/master/deploy/examples/object-bucket-claim-retain.yaml) Creates a request for a new bucket by referencing a StorageClass which saves the bucket when the initiating OBC is deleted. +* [object-bucket-claim-delete.yaml](https://github.com/rook/rook/blob/master/deploy/examples/object-bucket-claim-delete.yaml) Creates a request for a new bucket by referencing a StorageClass which deletes the bucket when the initiating OBC is deleted. +* [storageclass-bucket-retain.yaml](https://github.com/rook/rook/blob/master/deploy/examples/storageclass-bucket-retain.yaml) Creates a new StorageClass which defines the Ceph Object Store and retains the bucket after the initiating OBC is deleted. +* [storageclass-bucket-delete.yaml](https://github.com/rook/rook/blob/master/deploy/examples/storageclass-bucket-delete.yaml) Creates a new StorageClass which defines the Ceph Object Store and deletes the bucket after the initiating OBC is deleted. diff --git a/Documentation/Getting-Started/intro.md b/Documentation/Getting-Started/intro.md new file mode 120000 index 000000000000..32d46ee883b5 --- /dev/null +++ b/Documentation/Getting-Started/intro.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md new file mode 100644 index 000000000000..c5fd479abda2 --- /dev/null +++ b/Documentation/Getting-Started/quickstart.md @@ -0,0 +1,165 @@ +--- +title: Quickstart +--- + +Welcome to Rook! We hope you have a great experience installing the Rook **cloud-native storage orchestrator** platform to enable highly available, durable Ceph storage in your Kubernetes cluster. + +If you have any questions along the way, please don't hesitate to ask us in our [Slack channel](https://rook-io.slack.com). You can sign up for our Slack [here](https://slack.rook.io). + +This guide will walk you through the basic setup of a Ceph cluster and enable you to consume block, object, and file storage +from other pods running in your cluster. + +**Always use a virtual machine when testing Rook. Never use your host system where local devices may mistakenly be consumed.** + +## Minimum Version + +Kubernetes **v1.17** or higher is supported by Rook. + +## Prerequisites + +To make sure you have a Kubernetes cluster that is ready for `Rook`, you can [follow these instructions](Prerequisites/prerequisites.md). + +In order to configure the Ceph storage cluster, at least one of these local storage options are required: + +* Raw devices (no partitions or formatted filesystems) + * This requires `lvm2` to be installed on the host. + To avoid this dependency, you can create a single full-disk partition on the disk (see below) +* Raw partitions (no formatted filesystem) +* Persistent Volumes available from a storage class in `block` mode + +## TL;DR + +A simple Rook cluster can be created with the following kubectl commands and [example manifests](https://github.com/rook/rook/blob/master/deploy/examples). + +```console +$ git clone --single-branch --branch master https://github.com/rook/rook.git +cd rook/deploy/examples +kubectl create -f crds.yaml -f common.yaml -f operator.yaml +kubectl create -f cluster.yaml +``` + +After the cluster is running, you can create [block, object, or file](#storage) storage to be consumed by other applications in your cluster. + +## Deploy the Rook Operator + +The first step is to deploy the Rook operator. Check that you are using the [example yaml files](https://github.com/rook/rook/blob/master/deploy/examples) that correspond to your release of Rook. For more options, see the [example configurations documentation](example-configurations.md). + +```console +cd deploy/examples +kubectl create -f crds.yaml -f common.yaml -f operator.yaml + +# verify the rook-ceph-operator is in the `Running` state before proceeding +kubectl -n rook-ceph get pod +``` + +You can also deploy the operator with the [Rook Helm Chart](../Helm-Charts/operator-chart.md). + +Before you start the operator in production, there are some settings that you may want to consider: + +1. Consider if you want to enable certain Rook features that are disabled by default. See the [operator.yaml](https://github.com/rook/rook/blob/master/deploy/examples/operator.yaml) for these and other advanced settings. + 1. Device discovery: Rook will watch for new devices to configure if the `ROOK_ENABLE_DISCOVERY_DAEMON` setting is enabled, commonly used in bare metal clusters. + 2. Node affinity and tolerations: The CSI driver by default will run on any node in the cluster. To configure the CSI driver affinity, several settings are available. + +If you wish to deploy into a namespace other than the default `rook-ceph`, see the +[Ceph advanced configuration section](../Storage-Configuration/Advanced/ceph-configuration.md#using-alternate-namespaces) on the topic. + +## Cluster Environments + +The Rook documentation is focused around starting Rook in a production environment. Examples are also +provided to relax some settings for test environments. When creating the cluster later in this guide, consider these example cluster manifests: + +* [cluster.yaml](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml): Cluster settings for a production cluster running on bare metal. Requires at least three worker nodes. +* [cluster-on-pvc.yaml](https://github.com/rook/rook/blob/master/deploy/examples/cluster-on-pvc.yaml): Cluster settings for a production cluster running in a dynamic cloud environment. +* [cluster-test.yaml](https://github.com/rook/rook/blob/master/deploy/examples/cluster-test.yaml): Cluster settings for a test environment such as minikube. + +See the [Ceph example configurations](example-configurations.md) for more details. + +## Create a Ceph Cluster + +Now that the Rook operator is running we can create the Ceph cluster. For the cluster to survive reboots, +make sure you set the `dataDirHostPath` property that is valid for your hosts. For more settings, see the documentation on [configuring the cluster](../CRDs/ceph-cluster-crd.md). + +Create the cluster: + +```console +kubectl create -f cluster.yaml +``` + +Use `kubectl` to list pods in the `rook-ceph` namespace. You should be able to see the following pods once they are all running. +The number of osd pods will depend on the number of nodes in the cluster and the number of devices configured. +If you did not modify the `cluster.yaml` above, it is expected that one OSD will be created per node. + +!!! hint + If the `rook-ceph-mon`, `rook-ceph-mgr`, or `rook-ceph-osd` pods are not created, please refer to the + [Ceph common issues](../Troubleshooting/ceph-common-issues.md) for more details and potential solutions. + +```console +$ kubectl -n rook-ceph get pod +NAME READY STATUS RESTARTS AGE +csi-cephfsplugin-provisioner-d77bb49c6-n5tgs 5/5 Running 0 140s +csi-cephfsplugin-provisioner-d77bb49c6-v9rvn 5/5 Running 0 140s +csi-cephfsplugin-rthrp 3/3 Running 0 140s +csi-rbdplugin-hbsm7 3/3 Running 0 140s +csi-rbdplugin-provisioner-5b5cd64fd-nvk6c 6/6 Running 0 140s +csi-rbdplugin-provisioner-5b5cd64fd-q7bxl 6/6 Running 0 140s +rook-ceph-crashcollector-minikube-5b57b7c5d4-hfldl 1/1 Running 0 105s +rook-ceph-mgr-a-64cd7cdf54-j8b5p 1/1 Running 0 77s +rook-ceph-mon-a-694bb7987d-fp9w7 1/1 Running 0 105s +rook-ceph-mon-b-856fdd5cb9-5h2qk 1/1 Running 0 94s +rook-ceph-mon-c-57545897fc-j576h 1/1 Running 0 85s +rook-ceph-operator-85f5b946bd-s8grz 1/1 Running 0 92m +rook-ceph-osd-0-6bb747b6c5-lnvb6 1/1 Running 0 23s +rook-ceph-osd-1-7f67f9646d-44p7v 1/1 Running 0 24s +rook-ceph-osd-2-6cd4b776ff-v4d68 1/1 Running 0 25s +rook-ceph-osd-prepare-node1-vx2rz 0/2 Completed 0 60s +rook-ceph-osd-prepare-node2-ab3fd 0/2 Completed 0 60s +rook-ceph-osd-prepare-node3-w4xyz 0/2 Completed 0 60s +``` + +To verify that the cluster is in a healthy state, connect to the [Rook toolbox](../Troubleshooting/ceph-toolbox.md) and run the +`ceph status` command. + +* All mons should be in quorum +* A mgr should be active +* At least one OSD should be active +* If the health is not `HEALTH_OK`, the warnings or errors should be investigated + +```console +$ ceph status + cluster: + id: a0452c76-30d9-4c1a-a948-5d8405f19a7c + health: HEALTH_OK + + services: + mon: 3 daemons, quorum a,b,c (age 3m) + mgr: a(active, since 2m) + osd: 3 osds: 3 up (since 1m), 3 in (since 1m) +[]...] +``` + +If the cluster is not healthy, please refer to the [Ceph common issues](../Troubleshooting/ceph-common-issues.md) for more details and potential solutions. + +## Storage + +For a walkthrough of the three types of storage exposed by Rook, see the guides for: + +* **[Block](../Storage-Configuration/Block-Storage-RBD/block-storage.md)**: Create block storage to be consumed by a pod (RWO) +* **[Shared Filesystem](../Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md)**: Create a filesystem to be shared across multiple pods (RWX) +* **[Object](../Storage-Configuration/Object-Storage-RGW/object-storage.md)**: Create an object store that is accessible inside or outside the Kubernetes cluster + +## Ceph Dashboard + +Ceph has a dashboard in which you can view the status of your cluster. Please see the [dashboard guide](../Storage-Configuration/Monitoring/ceph-dashboard.md) for more details. + +## Tools + +Create a toolbox pod for full access to a ceph admin client for debugging and troubleshooting your Rook cluster. Please see the [toolbox documentation](../Troubleshooting/ceph-toolbox.md) for setup and usage information. Also see our [advanced configuration](../Storage-Configuration/Advanced/ceph-configuration.md) document for helpful maintenance and tuning examples. + +## Monitoring + +Each Rook cluster has some built in metrics collectors/exporters for monitoring with [Prometheus](https://prometheus.io/). +To learn how to set up monitoring for your Rook cluster, you can follow the steps in the [monitoring guide](../Storage-Configuration/Monitoring/ceph-monitoring.md). + +## Teardown + +When you are done with the test cluster, see [these instructions](../Storage-Configuration/ceph-teardown.md) to clean up the cluster. diff --git a/Documentation/ceph-storage.md b/Documentation/Getting-Started/storage-architecture.md similarity index 80% rename from Documentation/ceph-storage.md rename to Documentation/Getting-Started/storage-architecture.md index 19e632f836a6..05873f6f5178 100644 --- a/Documentation/ceph-storage.md +++ b/Documentation/Getting-Started/storage-architecture.md @@ -1,10 +1,7 @@ --- -title: Ceph Storage -weight: 2000 +title: Storage Architecture --- -# Ceph Storage - Ceph is a highly scalable distributed storage solution for **block storage**, **object storage**, and **shared filesystems** with years of production deployments. ## Design @@ -16,7 +13,7 @@ automates configuration of storage components and monitors the cluster to ensure and healthy. The Rook operator is a simple container that has all that is needed to bootstrap -and monitor the storage cluster. The operator will start and monitor [Ceph monitor pods](ceph-mon-health.md), the Ceph OSD daemons to provide RADOS storage, as well as start and manage other Ceph daemons. The operator manages CRDs for pools, object stores (S3/Swift), and filesystems by initializing the pods and other resources necessary to run the services. +and monitor the storage cluster. The operator will start and monitor [Ceph monitor pods](../Storage-Configuration/Advanced/ceph-mon-health.md), the Ceph OSD daemons to provide RADOS storage, as well as start and manage other Ceph daemons. The operator manages CRDs for pools, object stores (S3/Swift), and filesystems by initializing the pods and other resources necessary to run the services. The operator will monitor the storage daemons to ensure the cluster is healthy. Ceph mons will be started or failed over when necessary, and other adjustments are made as the cluster grows or shrinks. The operator will also watch for desired state changes @@ -24,7 +21,7 @@ specified in the Ceph custom resources (CRs) and apply the changes. Rook automatically configures the Ceph-CSI driver to mount the storage to your pods. -![Rook Components on Kubernetes](media/kubernetes.png) +![Rook Components on Kubernetes](ceph-storage/kubernetes.png) The `rook/ceph` image includes all necessary tools to manage the cluster. Rook is not in the Ceph data path. Many of the Ceph concepts like placement groups and crush maps diff --git a/Documentation/Helm-Charts/.pages b/Documentation/Helm-Charts/.pages new file mode 100644 index 000000000000..17d4548ca74c --- /dev/null +++ b/Documentation/Helm-Charts/.pages @@ -0,0 +1,5 @@ +nav: + - helm-charts.md + - operator-chart.md + - ceph-cluster-chart.md + - ... diff --git a/Documentation/helm-ceph-cluster.md b/Documentation/Helm-Charts/ceph-cluster-chart.md similarity index 66% rename from Documentation/helm-ceph-cluster.md rename to Documentation/Helm-Charts/ceph-cluster-chart.md index 5988b4268dac..9e31a94ba3df 100644 --- a/Documentation/helm-ceph-cluster.md +++ b/Documentation/Helm-Charts/ceph-cluster-chart.md @@ -1,25 +1,20 @@ --- -title: Ceph Cluster -weight: 10200 -indent: true +title: Ceph Cluster Helm Chart --- -{% include_relative branch.liquid %} - -# Ceph Cluster Helm Chart - Creates Rook resources to configure a [Ceph](https://ceph.io/) cluster using the [Helm](https://helm.sh) package manager. This chart is a simple packaging of templates that will optionally create Rook resources such as: -- CephCluster, CephFilesystem, and CephObjectStore CRs -- Storage classes to expose Ceph RBD volumes, CephFS volumes, and RGW buckets -- Ingress for external access to the dashboard -- Toolbox + +* CephCluster, CephFilesystem, and CephObjectStore CRs +* Storage classes to expose Ceph RBD volumes, CephFS volumes, and RGW buckets +* Ingress for external access to the dashboard +* Toolbox ## Prerequisites * Kubernetes 1.17+ * Helm 3.x -* Install the [Rook Operator chart](helm-operator.md) +* Install the [Rook Operator chart](operator-chart.md) ## Installing @@ -31,9 +26,10 @@ into the same namespace as the operator or a separate namespace. Rook currently publishes builds of this chart to the `release` and `master` channels. **Before installing, review the values.yaml to confirm if the default settings need to be updated.** + * If the operator was installed in a namespace other than `rook-ceph`, the namespace must be set in the `operatorNamespace` variable. -* Set the desired settings in the `cephClusterSpec`. The [defaults](https://github.com/rook/rook/tree/{{ branchName }}/deploy/charts/rook-ceph-cluster/values.yaml) +* Set the desired settings in the `cephClusterSpec`. The [defaults](https://github.com/rook/rook/tree/master/deploy/charts/rook-ceph-cluster/values.yaml) are only an example and not likely to apply to your cluster. * The `monitoring` section should be removed from the `cephClusterSpec`, as it is specified separately in the helm settings. * The default values for `cephBlockPools`, `cephFileSystems`, and `CephObjectStores` will create one of each, and their corresponding storage classes. @@ -43,7 +39,7 @@ Rook currently publishes builds of this chart to the `release` and `master` chan The release channel is the most recent release of Rook that is considered stable for the community. -The example install assumes you have first installed the [Rook Operator chart](helm-operator.md) +The example install assumes you have first installed the [Rook Operator Helm Chart](operator-chart.md) and created your customized values-override.yaml. ```console @@ -56,43 +52,43 @@ helm install --create-namespace --namespace rook-ceph rook-ceph-cluster \ The following tables lists the configurable parameters of the rook-operator chart and their default values. -| Parameter | Description | Default | -| ---------------------- | -------------------------------------------------------------------- | ---------------- | -| `operatorNamespace` | Namespace of the Rook Operator | `rook-ceph` | -| `kubeVersion` | Optional override of the target kubernetes version | `` | -| `configOverride` | Cluster ceph.conf override | | -| `toolbox.enabled` | Enable Ceph debugging pod deployment. See [toolbox](ceph-toolbox.md) | `false` | -| `toolbox.tolerations` | Toolbox tolerations | `[]` | -| `toolbox.affinity` | Toolbox affinity | `{}` | -| `toolbox.resources` | Toolbox resources | see values.yaml | -| `monitoring.enabled` | Enable Prometheus integration, will also create necessary RBAC rules | `false` | -| `monitoring.createPrometheusRules` | Whether to create the Prometheus rules for Ceph alerts | `false` | -| `cephClusterSpec.*` | Cluster configuration, see below | See below | -| `ingress.dashboard` | Enable an ingress for the ceph-dashboard | `{}` | -| `cephBlockPools.[*]` | A list of CephBlockPool configurations to deploy | See below | -| `cephFileSystems.[*]` | A list of CephFileSystem configurations to deploy | See below | -| `cephObjectStores.[*]` | A list of CephObjectStore configurations to deploy | See below | +| Parameter | Description | Default | +| ---------------------------------- | ------------------------------------------------------------------------------------------ | --------------- | +| `operatorNamespace` | Namespace of the Rook Operator | `rook-ceph` | +| `kubeVersion` | Optional override of the target kubernetes version | `` | +| `configOverride` | Cluster ceph.conf override | `` | +| `toolbox.enabled` | Enable Ceph debugging pod deployment. See [toolbox](../Troubleshooting/ceph-toolbox.md) | `false` | +| `toolbox.tolerations` | Toolbox tolerations | `[]` | +| `toolbox.affinity` | Toolbox affinity | `{}` | +| `toolbox.resources` | Toolbox resources | see values.yaml | +| `monitoring.enabled` | Enable Prometheus integration, will also create necessary RBAC rules | `false` | +| `monitoring.createPrometheusRules` | Whether to create the Prometheus rules for Ceph alerts | `false` | +| `cephClusterSpec.*` | Cluster configuration, see below | See below | +| `ingress.dashboard` | Enable an ingress for the ceph-dashboard | `{}` | +| `cephBlockPools.[*]` | A list of CephBlockPool configurations to deploy | See below | +| `cephFileSystems.[*]` | A list of CephFileSystem configurations to deploy | See below | +| `cephObjectStores.[*]` | A list of CephObjectStore configurations to deploy | See below | ### **Ceph Cluster Spec** The `CephCluster` CRD takes its spec from `cephClusterSpec.*`. This is not an exhaustive list of parameters. -For the full list, see the [Cluster CRD](ceph-cluster-crd.md) topic. +For the full list, see the [Cluster CRD](../CRDs/ceph-cluster-crd.md) topic. ### **Ceph Block Pools** The `cephBlockPools` array in the values file will define a list of CephBlockPool as described in the table below. -| Parameter | Description | Default | -| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | -| `name` | The name of the CephBlockPool | `ceph-blockpool` | -| `spec` | The CephBlockPool spec, see the [CephBlockPool](ceph-pool-crd.md#spec) documentation. | `{}` | -| `storageClass.enabled` | Whether a storage class is deployed alongside the CephBlockPool | `true` | -| `storageClass.isDefault` | Whether the storage class will be the default storage class for PVCs. See the PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) documentation for details. | `true` | -| `storageClass.name` | The name of the storage class | `ceph-block` | -| `storageClass.parameters` | See [Block Storage](ceph-block.md) documentation or the helm values.yaml for suitable values | see values.yaml | -| `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete` | -| `storageClass.allowVolumeExpansion` | Whether [volume expansion](https://kubernetes.io/docs/concepts/storage/storage-classes/#allow-volume-expansion) is allowed by default. | `true` | -| `storageClass.mountOptions` | Specifies the mount options for storageClass | `[]` | +| Parameter | Description | Default | +| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| `name` | The name of the CephBlockPool | `ceph-blockpool` | +| `spec` | The CephBlockPool spec, see the [CephBlockPool](../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) documentation. | `{}` | +| `storageClass.enabled` | Whether a storage class is deployed alongside the CephBlockPool | `true` | +| `storageClass.isDefault` | Whether the storage class will be the default storage class for PVCs. See [PersistentVolumeClaim documentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) for details. | `true` | +| `storageClass.name` | The name of the storage class | `ceph-block` | +| `storageClass.parameters` | See [Block Storage](../Storage-Configuration/Block-Storage-RBD/block-storage.md) documentation or the helm values.yaml for suitable values | see values.yaml | +| `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete` | +| `storageClass.allowVolumeExpansion` | Whether [volume expansion](https://kubernetes.io/docs/concepts/storage/storage-classes/#allow-volume-expansion) is allowed by default. | `true` | +| `storageClass.mountOptions` | Specifies the mount options for storageClass | `[]` | ### **Ceph File Systems** @@ -101,11 +97,11 @@ The `cephFileSystems` array in the values file will define a list of CephFileSys | Parameter | Description | Default | | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | | `name` | The name of the CephFileSystem | `ceph-filesystem` | -| `spec` | The CephFileSystem spec, see the [CephFilesystem CRD](ceph-filesystem-crd.md) documentation. | see values.yaml | +| `spec` | The CephFileSystem spec, see the [CephFilesystem CRD](../CRDs/Shared-Filesystem/ceph-filesystem-crd.md) documentation. | see values.yaml | | `storageClass.enabled` | Whether a storage class is deployed alongside the CephFileSystem | `true` | | `storageClass.name` | The name of the storage class | `ceph-filesystem` | -| `storageClass.pool` | The name of [Data Pool](ceph-filesystem-crd.md#pools), without the filesystem name prefix | `data0` | -| `storageClass.parameters` | See [Shared Filesystem](ceph-filesystem.md) documentation or the helm values.yaml for suitable values | see values.yaml | +| `storageClass.pool` | The name of [Data Pool](../CRDs/Shared-Filesystem/ceph-filesystem-crd.md#pools), without the filesystem name prefix | `data0` | +| `storageClass.parameters` | See [Shared Filesystem](../Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md) documentation or the helm values.yaml for suitable values | see values.yaml | | `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete` | | `storageClass.mountOptions` | Specifies the mount options for storageClass | `[]` | @@ -113,14 +109,14 @@ The `cephFileSystems` array in the values file will define a list of CephFileSys The `cephObjectStores` array in the values file will define a list of CephObjectStore as described in the table below. -| Parameter | Description | Default | -| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -| `name` | The name of the CephObjectStore | `ceph-objectstore` | -| `spec` | The CephObjectStore spec, see the [CephObjectStore CRD](ceph-object-store-crd.md) documentation. | see values.yaml | -| `storageClass.enabled` | Whether a storage class is deployed alongside the CephObjectStore | `true` | -| `storageClass.name` | The name of the storage class | `ceph-bucket` | -| `storageClass.parameters` | See [Object Store storage class](ceph-object-bucket-claim.md) documentation or the helm values.yaml for suitable values | see values.yaml | -| `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete` | +| Parameter | Description | Default | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------ | +| `name` | The name of the CephObjectStore | `ceph-objectstore` | +| `spec` | The CephObjectStore spec, see the [CephObjectStore CRD](../CRDs/Object-Storage/ceph-object-store-crd.md) documentation. | see values.yaml | +| `storageClass.enabled` | Whether a storage class is deployed alongside the CephObjectStore | `true` | +| `storageClass.name` | The name of the storage class | `ceph-bucket` | +| `storageClass.parameters` | See [Object Store storage class](../Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md) documentation or the helm values.yaml for suitable values | see values.yaml | +| `storageClass.reclaimPolicy` | The default [Reclaim Policy](https://kubernetes.io/docs/concepts/storage/storage-classes/#reclaim-policy) to apply to PVCs created with this storage class. | `Delete` | ### **Existing Clusters** @@ -172,4 +168,4 @@ The command removes all the Kubernetes components associated with the chart and chart does not remove the Rook operator. In addition, all data on hosts in the Rook data directory (`/var/lib/rook` by default) and on OSD raw devices is kept. To reuse disks, you will have to wipe them before recreating the cluster. -See the [teardown documentation](ceph-teardown.md) for more information. +See the [teardown documentation](../Storage-Configuration/ceph-teardown.md) for more information. diff --git a/Documentation/Helm-Charts/helm-charts.md b/Documentation/Helm-Charts/helm-charts.md new file mode 100644 index 000000000000..c430fd38c1bc --- /dev/null +++ b/Documentation/Helm-Charts/helm-charts.md @@ -0,0 +1,13 @@ +--- +title: Helm Charts Overview +--- + +Rook has published the following Helm charts for the Ceph storage provider: + +* [Rook Ceph Operator](operator-chart.md): Starts the Ceph Operator, which will watch for Ceph CRs (custom resources) +* [Rook Ceph Cluster](ceph-cluster-chart.md): Creates Ceph CRs that the operator will use to configure the cluster + +The Helm charts are intended to simplify deployment and upgrades. +Configuring the Rook resources without Helm is also fully supported by creating the +[manifests](https://github.com/rook/rook/tree/master/deploy/examples) +directly. diff --git a/Documentation/helm-operator.md b/Documentation/Helm-Charts/operator-chart.md similarity index 51% rename from Documentation/helm-operator.md rename to Documentation/Helm-Charts/operator-chart.md index 606f8d7e9a0a..490ac1fc4f75 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -1,13 +1,7 @@ --- -title: Ceph Operator -weight: 10100 -indent: true +title: Ceph Operator Helm Chart --- -{% include_relative branch.liquid %} - -# Ceph Operator Helm Chart - Installs [rook](https://github.com/rook/rook) to create, configure, and manage Ceph clusters on Kubernetes. ## Introduction @@ -24,8 +18,9 @@ See the [Helm support matrix](https://helm.sh/docs/topics/version_skew/) for mor ## Installing The Ceph Operator helm chart will install the basic components necessary to create a storage platform for your Kubernetes cluster. + 1. Install the Helm chart -1. [Create a Rook cluster](quickstart.md#create-a-rook-cluster). +1. [Create a Rook cluster](../Getting-Started/quickstart.md#create-a-ceph-cluster). The `helm install` command deploys rook on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the `rook-ceph` namespace (you will install your clusters into separate namespaces). @@ -40,103 +35,102 @@ helm repo add rook-release https://charts.rook.io/release helm install --create-namespace --namespace rook-ceph rook-ceph rook-release/rook-ceph -f values.yaml ``` -For example settings, see the next section or [values.yaml](https://github.com/rook/rook/tree/{{ branchName }}/deploy/charts/rook-ceph/values.yaml) +For example settings, see the next section or [values.yaml](https://github.com/rook/rook/tree/master/deploy/charts/rook-ceph/values.yaml) ## Configuration The following tables lists the configurable parameters of the rook-operator chart and their default values. -| Parameter | Description | Default | -| ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | -| `image.repository` | Image | `rook/ceph` | -| `image.tag` | Image tag | `master` | -| `image.pullPolicy` | Image pull policy | `IfNotPresent` | -| `crds.enabled` | If true, the helm chart will create the Rook CRDs. Do NOT change to `false` in a running cluster or CRs will be deleted! | `true` | -| `rbacEnable` | If true, create & use RBAC resources | `true` | -| `pspEnable` | If true, create & use PSP resources | `true` | -| `resources` | Pod resource requests & limits | `{}` | -| `annotations` | Pod annotations | `{}` | -| `logLevel` | Global log level | `INFO` | -| `nodeSelector` | Kubernetes `nodeSelector` to add to the Deployment. | | -| `tolerations` | List of Kubernetes `tolerations` to add to the Deployment. | `[]` | -| `unreachableNodeTolerationSeconds` | Delay to use for the node.kubernetes.io/unreachable pod failure toleration to override the Kubernetes default of 5 minutes | `5s` | -| `currentNamespaceOnly` | Whether the operator should watch cluster CRD in its own namespace or not | `false` | -| `hostpathRequiresPrivileged` | Runs Ceph Pods as privileged to be able to write to `hostPath`s in OpenShift with SELinux restrictions. | `false` | -| `discover.priorityClassName` | The priority class name to add to the discover pods | | -| `discover.toleration` | Toleration for the discover pods | | -| `discover.tolerationKey` | The specific key of the taint to tolerate | | -| `discover.tolerations` | Array of tolerations in YAML format which will be added to discover deployment | | -| `discover.nodeAffinity` | The node labels for affinity of `discover-agent` (***) | | -| `discover.podLabels` | Labels to add to the discover pods. | | -| `csi.enableRbdDriver` | Enable Ceph CSI RBD driver. | `true` | -| `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver. | `true` | -| `csi.enableCephfsSnapshotter` | Enable Snapshotter in CephFS provisioner pod. | `true` | -| `csi.enableRBDSnapshotter` | Enable Snapshotter in RBD provisioner pod. | `true` | -| `csi.pluginPriorityClassName` | PriorityClassName to be set on csi driver plugin pods. | | -| `csi.provisionerPriorityClassName` | PriorityClassName to be set on csi driver provisioner pods. | | -| `csi.enableOMAPGenerator` | EnableOMAP generator deploys omap sidecar in CSI provisioner pod, to enable it set it to true | `false` | -| `csi.rbdFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted | ReadWriteOnceWithFSType | -| `csi.cephFSFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted | ReadWriteOnceWithFSType | -| `csi.nfsFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted | ReadWriteOnceWithFSType | -| `csi.logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | -| `csi.grpcTimeoutInSeconds` | Set GRPC timeout for csi containers. | `150` | -| `csi.provisionerReplicas` | Set replicas for csi provisioner deployment. | `2` | -| `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics. | `false` | -| `csi.enableCSIHostNetwork` | Enable Host Networking for Ceph CSI nodeplugins. | `false` | -| `csi.enablePluginSelinuxHostMount` | Enable Host mount for /etc/selinux directory for Ceph CSI nodeplugins. | `false` | -| `csi.enableCSIEncryption` | Enable Ceph CSI PVC encryption support. | `false` | -| `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment. | | -| `csi.provisionerNodeAffinity` | The node labels for affinity of the CSI provisioner deployment (***) | | -| `csi.pluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet | | -| `csi.pluginNodeAffinity` | The node labels for affinity of the CephCSI plugin DaemonSet (***) | | -| `csi.rbdProvisionerTolerations` | Array of tolerations in YAML format which will be added to CephCSI RBD provisioner deployment. | | -| `csi.rbdProvisionerNodeAffinity` | The node labels for affinity of the CephCSI RBD provisioner deployment (***) | | -| `csi.rbdPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI RBD plugin DaemonSet | | -| `csi.rbdPluginNodeAffinity` | The node labels for affinity of the CephCSI RBD plugin DaemonSet (***) | | -| `csi.cephFSProvisionerTolerations` | Array of tolerations in YAML format which will be added to CephCSI CephFS provisioner deployment. | | -| `csi.cephFSProvisionerNodeAffinity` | The node labels for affinity of the CephCSI CephFS provisioner deployment (***) | | -| `csi.cephFSPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI CephFS plugin DaemonSet | | -| `csi.cephFSPluginNodeAffinity` | The node labels for affinity of the CephCSI CephFS plugin DaemonSet (***) | | -| `csi.nfsProvisionerTolerations` | Array of tolerations in YAML format which will be added to CephCSI NFS provisioner deployment. | | -| `csi.nfsProvisionerNodeAffinity` | The node labels for affinity of the CephCSI NFS provisioner deployment (***) | | -| `csi.nfsPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI NFS plugin DaemonSet | | -| `csi.nfsPluginNodeAffinity` | The node labels for affinity of the CephCSI NFS plugin DaemonSet (***) | | -| `csi.csiRBDProvisionerResource` | CEPH CSI RBD provisioner resource requirement list. | | -| `csi.csiRBDPluginResource` | CEPH CSI RBD plugin resource requirement list. | | -| `csi.csiCephFSProvisionerResource` | CEPH CSI CephFS provisioner resource requirement list. | | -| `csi.csiCephFSPluginResource` | CEPH CSI CephFS plugin resource requirement list. | | -| `csi.csiNFSProvisionerResource` | CEPH CSI NFS provisioner resource requirement list. | | -| `csi.csiNFSPluginResource` | CEPH CSI NFS plugin resource requirement list. | | -| `csi.cephfsGrpcMetricsPort` | CSI CephFS driver GRPC metrics port. | `9091` | -| `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port. | `9081` | -| `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` | -| `csi.csiAddonsPort` | CSI Addons server port. | `9070` | -| `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port. | `8080` | -| `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` | -| `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | -| `csi.cephcsi.image` | Ceph CSI image. | `quay.io/cephcsi/cephcsi:v3.6.1` | -| `csi.rbdPluginUpdateStrategy` | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | -| `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | -| `csi.nfsPluginUpdateStrategy` | CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | -| `csi.registrar.image` | Kubernetes CSI registrar image. | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0` | -| `csi.resizer.image` | Kubernetes CSI resizer image. | `k8s.gcr.io/sig-storage/csi-resizer:v1.4.0` | -| `csi.provisioner.image` | Kubernetes CSI provisioner image. | `k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0` | -| `csi.snapshotter.image` | Kubernetes CSI snapshotter image. | `k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1` | -| `csi.attacher.image` | Kubernetes CSI Attacher image. | `k8s.gcr.io/sig-storage/csi-attacher:v3.4.0` | -| `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Pods. | | -| `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | | -| `csi.volumeReplication.enabled` | Enable Volume Replication. | `false` | -| `csi.volumeReplication.image` | Volume Replication Controller image. | `quay.io/csiaddons/volumereplication-operator:v0.3.0` | -| `csi.csiAddons.enabled` | Enable CSIAddons | `false` | -| `csi.csiAddons.image` | CSIAddons Sidecar image. | `quay.io/csiaddons/k8s-sidecar:v0.2.1` | -| `csi.nfs.enabled` | Enable nfs driver. | `false` | -| `csi.nfs.image` | NFS nodeplugin image. | `k8s.gcr.io/sig-storage/nfsplugin:v3.1.0` | -| `admissionController.tolerations` | Array of tolerations in YAML format which will be added to admission controller deployment. | | -| `admissionController.nodeAffinity` | The node labels for affinity of the admission controller deployment (***) | | -| `monitoring.enabled` | Create necessary RBAC rules for Rook to integrate with Prometheus monitoring in the operator namespace. Requires Prometheus to be pre-installed. | `false` | - -* * * `nodeAffinity` and `*NodeAffinity` options should have the format `"role=storage,rook; storage=ceph"` or `storage=;role=rook-example` or `storage=;` (_checks only for presence of key_) - +| Parameter | Description | Default | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------- | +| `image.repository` | Image | `rook/ceph` | +| `image.tag` | Image tag | `master` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `crds.enabled` | If true, the helm chart will create the Rook CRDs. Do NOT change to `false` in a running cluster or CRs will be deleted! | `true` | +| `rbacEnable` | If true, create & use RBAC resources | `true` | +| `pspEnable` | If true, create & use PSP resources | `true` | +| `resources` | Pod resource requests & limits | `{}` | +| `annotations` | Pod annotations | `{}` | +| `logLevel` | Global log level | `INFO` | +| `nodeSelector` | Kubernetes `nodeSelector` to add to the Deployment. | `` | +| `tolerations` | List of Kubernetes `tolerations` to add to the Deployment. | `[]` | +| `unreachableNodeTolerationSeconds` | Delay to use for the node.kubernetes.io/unreachable pod failure toleration to override the Kubernetes default of 5 minutes | `5s` | +| `currentNamespaceOnly` | Whether the operator should watch cluster CRD in its own namespace or not | `false` | +| `hostpathRequiresPrivileged` | Runs Ceph Pods as privileged to be able to write to `hostPath`s in OpenShift with SELinux restrictions. | `false` | +| `discover.priorityClassName` | The priority class name to add to the discover pods | `` | +| `discover.toleration` | Toleration for the discover pods | `` | +| `discover.tolerationKey` | The specific key of the taint to tolerate | `` | +| `discover.tolerations` | Array of tolerations in YAML format which will be added to discover deployment | `` | +| `discover.nodeAffinity` | The node labels for affinity of `discover-agent` [^1] | `` | +| `discover.podLabels` | Labels to add to the discover pods. | `` | +| `csi.enableRbdDriver` | Enable Ceph CSI RBD driver. | `true` | +| `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver. | `true` | +| `csi.enableCephfsSnapshotter` | Enable Snapshotter in CephFS provisioner pod. | `true` | +| `csi.enableRBDSnapshotter` | Enable Snapshotter in RBD provisioner pod. | `true` | +| `csi.pluginPriorityClassName` | PriorityClassName to be set on csi driver plugin pods. | `` | +| `csi.provisionerPriorityClassName` | PriorityClassName to be set on csi driver provisioner pods. | `` | +| `csi.enableOMAPGenerator` | EnableOMAP generator deploys omap sidecar in CSI provisioner pod, to enable it set it to true | `false` | +| `csi.rbdFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted | `ReadWriteOnceWithFSType` | +| `csi.cephFSFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted | `ReadWriteOnceWithFSType` | +| `csi.nfsFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted | `ReadWriteOnceWithFSType` | +| `csi.logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | +| `csi.grpcTimeoutInSeconds` | Set GRPC timeout for csi containers. | `150` | +| `csi.provisionerReplicas` | Set replicas for csi provisioner deployment. | `2` | +| `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics. | `false` | +| `csi.enableCSIHostNetwork` | Enable Host Networking for Ceph CSI nodeplugins. | `false` | +| `csi.enablePluginSelinuxHostMount` | Enable Host mount for /etc/selinux directory for Ceph CSI nodeplugins. | `false` | +| `csi.enableCSIEncryption` | Enable Ceph CSI PVC encryption support. | `false` | +| `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment. | `` | +| `csi.provisionerNodeAffinity` | The node labels for affinity of the CSI provisioner deployment [^1] | `` | +| `csi.pluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet | `` | +| `csi.pluginNodeAffinity` | The node labels for affinity of the CephCSI plugin DaemonSet [^1] | `` | +| `csi.rbdProvisionerTolerations` | Array of tolerations in YAML format which will be added to CephCSI RBD provisioner deployment. | `` | +| `csi.rbdProvisionerNodeAffinity` | The node labels for affinity of the CephCSI RBD provisioner deployment [^1] | `` | +| `csi.rbdPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI RBD plugin DaemonSet | `` | +| `csi.rbdPluginNodeAffinity` | The node labels for affinity of the CephCSI RBD plugin DaemonSet [^1] | `` | +| `csi.cephFSProvisionerTolerations` | Array of tolerations in YAML format which will be added to CephCSI CephFS provisioner deployment. | `` | +| `csi.cephFSProvisionerNodeAffinity` | The node labels for affinity of the CephCSI CephFS provisioner deployment [^1] | `` | +| `csi.cephFSPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI CephFS plugin DaemonSet | `` | +| `csi.cephFSPluginNodeAffinity` | The node labels for affinity of the CephCSI CephFS plugin DaemonSet [^1] | `` | +| `csi.nfsProvisionerTolerations` | Array of tolerations in YAML format which will be added to CephCSI NFS provisioner deployment. | `` | +| `csi.nfsProvisionerNodeAffinity` | The node labels for affinity of the CephCSI NFS provisioner deployment [^1] | `` | +| `csi.nfsPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI NFS plugin DaemonSet | `` | +| `csi.nfsPluginNodeAffinity` | The node labels for affinity of the CephCSI NFS plugin DaemonSet [^1] | `` | +| `csi.csiRBDProvisionerResource` | CEPH CSI RBD provisioner resource requirement list. | `` | +| `csi.csiRBDPluginResource` | CEPH CSI RBD plugin resource requirement list. | `` | +| `csi.csiCephFSProvisionerResource` | CEPH CSI CephFS provisioner resource requirement list. | `` | +| `csi.csiCephFSPluginResource` | CEPH CSI CephFS plugin resource requirement list. | `` | +| `csi.csiNFSProvisionerResource` | CEPH CSI NFS provisioner resource requirement list. | `` | +| `csi.csiNFSPluginResource` | CEPH CSI NFS plugin resource requirement list. | `` | +| `csi.cephfsGrpcMetricsPort` | CSI CephFS driver GRPC metrics port. | `9091` | +| `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port. | `9081` | +| `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` | +| `csi.csiAddonsPort` | CSI Addons server port. | `9070` | +| `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port. | `8080` | +| `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` | +| `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | +| `csi.cephcsi.image` | Ceph CSI image. | `quay.io/cephcsi/cephcsi:v3.6.1` | +| `csi.rbdPluginUpdateStrategy` | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | +| `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | +| `csi.nfsPluginUpdateStrategy` | CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | +| `csi.registrar.image` | Kubernetes CSI registrar image. | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0` | +| `csi.resizer.image` | Kubernetes CSI resizer image. | `k8s.gcr.io/sig-storage/csi-resizer:v1.4.0` | +| `csi.provisioner.image` | Kubernetes CSI provisioner image. | `k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0` | +| `csi.snapshotter.image` | Kubernetes CSI snapshotter image. | `k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1` | +| `csi.attacher.image` | Kubernetes CSI Attacher image. | `k8s.gcr.io/sig-storage/csi-attacher:v3.4.0` | +| `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Pods. | `` | +| `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | `` | +| `csi.volumeReplication.enabled` | Enable Volume Replication. | `false` | +| `csi.volumeReplication.image` | Volume Replication Controller image. | `quay.io/csiaddons/volumereplication-operator:v0.3.0` | +| `csi.csiAddons.enabled` | Enable CSIAddons | `false` | +| `csi.csiAddons.image` | CSIAddons Sidecar image. | `quay.io/csiaddons/k8s-sidecar:v0.2.1` | +| `csi.nfs.enabled` | Enable nfs driver. | `false` | +| `csi.nfs.image` | NFS nodeplugin image. | `k8s.gcr.io/sig-storage/nfsplugin:v3.1.0` | +| `admissionController.tolerations` | Array of tolerations in YAML format which will be added to admission controller deployment. | `` | +| `admissionController.nodeAffinity` | The node labels for affinity of the admission controller deployment [^1] | `` | +| `monitoring.enabled` | Create necessary RBAC rules for Rook to integrate with Prometheus monitoring in the operator namespace. Requires Prometheus to be pre-installed. | `false` | + +[^1]: `nodeAffinity` and `*NodeAffinity` options should have the format `"role=storage,rook; storage=ceph"` or `storage=;role=rook-example` or `storage=;` (_checks only for presence of key_) ### **Development Build** @@ -167,4 +161,4 @@ helm delete --namespace rook-ceph rook-ceph The command removes all the Kubernetes components associated with the chart and deletes the release. -After uninstalling you may want to clean up the CRDs as described on the [teardown documentation](ceph-teardown.md#removing-the-cluster-crd-finalizer). +After uninstalling you may want to clean up the CRDs as described on the [teardown documentation](../Storage-Configuration/ceph-teardown.md#removing-the-cluster-crd-finalizer). diff --git a/Documentation/README.md b/Documentation/README.md index e57ddd93a84c..98db404f255d 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -1,3 +1,7 @@ +--- +title: Rook +--- + # Rook Rook is an open source **cloud-native storage orchestrator**, providing the platform, framework, and support for a diverse set of storage solutions to natively integrate with cloud-native environments. @@ -15,7 +19,7 @@ See our [Quickstart](quickstart.md) guide to get started with the Ceph operator! ## Designs -[Ceph](https://docs.ceph.com/en/latest/) is a highly scalable distributed storage solution for block storage, object storage, and shared filesystems with years of production deployments. See the [Ceph overview](ceph-storage.md). +[Ceph](https://docs.ceph.com/en/latest/) is a highly scalable distributed storage solution for block storage, object storage, and shared filesystems with years of production deployments. See the [Ceph overview](storage-architecture.md). For detailed design documentation, see also the [design docs](https://github.com/rook/rook/tree/master/design). diff --git a/Documentation/Storage-Configuration/.pages b/Documentation/Storage-Configuration/.pages new file mode 100644 index 000000000000..df8838dd13ab --- /dev/null +++ b/Documentation/Storage-Configuration/.pages @@ -0,0 +1,9 @@ +nav: + - Block-Storage-RBD + - Shared-Filesystem-CephFS + - Object-Storage-RGW + - Monitoring + - Ceph-CSI + - ceph-teardown.md + - Advanced + - ... diff --git a/Documentation/Storage-Configuration/Advanced/.pages b/Documentation/Storage-Configuration/Advanced/.pages new file mode 100644 index 000000000000..bdb5c55d8361 --- /dev/null +++ b/Documentation/Storage-Configuration/Advanced/.pages @@ -0,0 +1,7 @@ +nav: + - ceph-configuration.md + - configuration.md + - key-management-system.md + - ceph-osd-mgmt.md + - ceph-mon-health.md + - ... diff --git a/Documentation/ceph-advanced-configuration.md b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md similarity index 75% rename from Documentation/ceph-advanced-configuration.md rename to Documentation/Storage-Configuration/Advanced/ceph-configuration.md index 4ee945cf214e..c4facbb0fbd9 100644 --- a/Documentation/ceph-advanced-configuration.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md @@ -1,33 +1,14 @@ --- -title: Advanced Configuration -weight: 11300 -indent: true +title: Ceph Configuration --- -{% include_relative branch.liquid %} - -# Advanced Configuration These examples show how to perform advanced configuration tasks on your Rook storage cluster. -* [Prerequisites](#prerequisites) -* [Using alternate namespaces](#using-alternate-namespaces) -* [Deploying a second cluster](#deploying-a-second-cluster) -* [Log Collection](#log-collection) -* [OSD Information](#osd-information) -* [Separate Storage Groups](#separate-storage-groups) -* [Configuring Pools](#configuring-pools) -* [Custom ceph.conf Settings](#custom-cephconf-settings) -* [Custom CSI ceph.conf Settings](#custom-csi-cephconf-settings) -* [OSD CRUSH Settings](#osd-crush-settings) -* [OSD Dedicated Network](#osd-dedicated-network) -* [Phantom OSD Removal](#phantom-osd-removal) -* [Auto Expansion of OSDs](#auto-expansion-of-osds) - ## Prerequisites Most of the examples make use of the `ceph` client command. A quick way to use -the Ceph client suite is from a [Rook Toolbox container](ceph-toolbox.md). +the Ceph client suite is from a [Rook Toolbox container](../../Troubleshooting/ceph-toolbox.md). The Kubernetes based examples assume Rook OSD pods are in the `rook-ceph` namespace. If you run them in a different namespace, modify `kubectl -n rook-ceph [...]` to fit @@ -45,7 +26,7 @@ Ceph cluster if you only wish to have one Operator manage all Ceph clusters. This will help you manage namespaces more easily, but you should still make sure the resources are configured to your liking. -```sh +```console cd deploy/examples export ROOK_OPERATOR_NAMESPACE="rook-ceph" @@ -68,7 +49,7 @@ kubectl apply -f common.yaml -f operator.yaml -f cluster.yaml # add other files If you wish to create a new CephCluster in a different namespace than `rook-ceph` while using a single operator to manage both clusters execute the following: -```sh +```console cd deploy/examples NAMESPACE=rook-ceph-secondary envsubst < common-second-cluster.yaml | kubectl create -f - @@ -127,41 +108,41 @@ done The output should look something like this. ->``` ->Pod: osd-m2fz2 ->Node: node1.zbrbdl ->-osd0 sda3 557.3G bluestore ->-osd1 sdf3 110.2G bluestore ->-osd2 sdd3 277.8G bluestore ->-osd3 sdb3 557.3G bluestore ->-osd4 sde3 464.2G bluestore ->-osd5 sdc3 557.3G bluestore -> ->Pod: osd-nxxnq ->Node: node3.zbrbdl ->-osd6 sda3 110.7G bluestore ->-osd17 sdd3 1.8T bluestore ->-osd18 sdb3 231.8G bluestore ->-osd19 sdc3 231.8G bluestore -> ->Pod: osd-tww1h ->Node: node2.zbrbdl ->-osd7 sdc3 464.2G bluestore ->-osd8 sdj3 557.3G bluestore ->-osd9 sdf3 66.7G bluestore ->-osd10 sdd3 464.2G bluestore ->-osd11 sdb3 147.4G bluestore ->-osd12 sdi3 557.3G bluestore ->-osd13 sdk3 557.3G bluestore ->-osd14 sde3 66.7G bluestore ->-osd15 sda3 110.2G bluestore ->-osd16 sdh3 135.1G bluestore ->``` - +```console +Pod: osd-m2fz2 +Node: node1.zbrbdl +-osd0 sda3 557.3G bluestore +-osd1 sdf3 110.2G bluestore +-osd2 sdd3 277.8G bluestore +-osd3 sdb3 557.3G bluestore +-osd4 sde3 464.2G bluestore +-osd5 sdc3 557.3G bluestore + +Pod: osd-nxxnq +Node: node3.zbrbdl +-osd6 sda3 110.7G bluestore +-osd17 sdd3 1.8T bluestore +-osd18 sdb3 231.8G bluestore +-osd19 sdc3 231.8G bluestore + +Pod: osd-tww1h +Node: node2.zbrbdl +-osd7 sdc3 464.2G bluestore +-osd8 sdj3 557.3G bluestore +-osd9 sdf3 66.7G bluestore +-osd10 sdd3 464.2G bluestore +-osd11 sdb3 147.4G bluestore +-osd12 sdi3 557.3G bluestore +-osd13 sdk3 557.3G bluestore +-osd14 sde3 66.7G bluestore +-osd15 sda3 110.2G bluestore +-osd16 sdh3 135.1G bluestore +``` ## Separate Storage Groups -> **DEPRECATED**: Instead of manually needing to set this, the `deviceClass` property can be used on Pool structures in `CephBlockPool`, `CephFilesystem` and `CephObjectStore` CRD objects. +!!! attention + It is **deprecated to manually need to set this**, the `deviceClass` property can be used on Pool structures in `CephBlockPool`, `CephFilesystem` and `CephObjectStore` CRD objects. By default Rook/Ceph puts all storage under one replication rule in the CRUSH Map which provides the maximum amount of storage capacity for a cluster. If you @@ -176,9 +157,10 @@ a common practice for those looking to target certain workloads onto faster ### Placement Group Sizing -> **NOTE**: Since Ceph Nautilus (v14.x), you can use the Ceph MGR `pg_autoscaler` -> module to auto scale the PGs as needed. If you want to enable this feature, -> please refer to [Default PG and PGP counts](ceph-configuration.md#default-pg-and-pgp-counts). +!!! note + Since Ceph Nautilus (v14.x), you can use the Ceph MGR `pg_autoscaler` + module to auto scale the PGs as needed. If you want to enable this feature, + please refer to [Default PG and PGP counts](ceph-configuration.md#default-pg-and-pgp-counts). The general rules for deciding how many PGs your pool(s) should contain is: @@ -208,13 +190,14 @@ before changing the number of PGs. ceph osd pool set rbd pg_num 512 ``` -## Custom ceph.conf Settings +## Custom `ceph.conf` Settings -> **WARNING**: The advised method for controlling Ceph configuration is to manually use the Ceph CLI -> or the Ceph dashboard because this offers the most flexibility. It is highly recommended that this -> only be used when absolutely necessary and that the `config` be reset to an empty string if/when the -> configurations are no longer necessary. Configurations in the config file will make the Ceph cluster -> less configurable from the CLI and dashboard and may make future tuning or debugging difficult. +!!! warning + The advised method for controlling Ceph configuration is to manually use the Ceph CLI + or the Ceph dashboard because this offers the most flexibility. It is highly recommended that this + only be used when absolutely necessary and that the `config` be reset to an empty string if/when the + configurations are no longer necessary. Configurations in the config file will make the Ceph cluster + less configurable from the CLI and dashboard and may make future tuning or debugging difficult. Setting configs via Ceph's CLI requires that at least one mon be available for the configs to be set, and setting configs via dashboard requires at least one mgr to be available. Ceph may also have @@ -224,8 +207,9 @@ configuration settings, the `rook-config-override` ConfigMap exists, and the `co set with the contents of a `ceph.conf` file. The contents will be propagated to all mon, mgr, OSD, MDS, and RGW daemons as an `/etc/ceph/ceph.conf` file. -> **WARNING**: Rook performs no validation on the config, so the validity of the settings is the -> user's responsibility. +!!! warning + Rook performs no validation on the config, so the validity of the settings is the + user's responsibility. If the `rook-config-override` ConfigMap is created before the cluster is started, the Ceph daemons will automatically pick up the settings. If you add the settings to the ConfigMap after the cluster @@ -244,7 +228,7 @@ cluster's namespace is created before the cluster is created, the daemons will p at first launch. To automate the restart of the Ceph daemon pods, you will need to trigger an update to the pod specs. -The simplest way to trigger the update is to add [annotations or labels](ceph-cluster-crd.md#annotations-and-labels) +The simplest way to trigger the update is to add [annotations or labels](../../CRDs/ceph-cluster-crd.md#annotations-and-labels) to the CephCluster CR for the daemons you want to restart. The operator will then proceed with a rolling update, similar to any other update to the cluster. @@ -253,8 +237,9 @@ update, similar to any other update to the cluster. In this example we will set the default pool `size` to two, and tell OSD daemons not to change the weight of OSDs on startup. -> **WARNING**: Modify Ceph settings carefully. You are leaving the sandbox tested by Rook. -> Changing the settings could result in unhealthy daemons or even data loss if used incorrectly. +!!! warning + Modify Ceph settings carefully. You are leaving the sandbox tested by Rook. + Changing the settings could result in unhealthy daemons or even data loss if used incorrectly. When the Rook Operator creates a cluster, a placeholder ConfigMap is created that will allow you to override Ceph configuration settings. When the daemon pods are started, the @@ -300,12 +285,13 @@ data: osd pool default size = 2 ``` -## Custom CSI ceph.conf Settings +## Custom CSI `ceph.conf` Settings -> **WARNING**: It is highly recommended to use the default setting that comes with -> CephCSI and this can only be used when absolutely necessary. -> The `ceph.conf` should be reset back to default values if/when the configurations are no -> longer necessary. +!!! warning + It is highly recommended to use the default setting that comes with + CephCSI and this can only be used when absolutely necessary. + The `ceph.conf` should be reset back to default values if/when the configurations are no + longer necessary. If the `csi-ceph-conf-override` ConfigMap is created before the cluster is started, the CephCSI pods will automatically pick up the settings. If you @@ -315,14 +301,15 @@ to take immediate effect. After the CSI pods are restarted, the new settings should be in effect. -### Example +### Example CSI `ceph.conf` Settings -In this [Example](https://github.com/rook/rook/tree/{{ branchName }}/deploy/csi-ceph-conf-override.yaml) we +In this [Example](https://github.com/rook/rook/tree/master/deploy/csi-ceph-conf-override.yaml) we will set the `rbd_validate_pool` to `false` to skip rbd pool validation. -> **WARNING**: Modify Ceph settings carefully to avoid modifying the default -> configuration. -> Changing the settings could result in unexpected results if used incorrectly. +!!! warning + Modify Ceph settings carefully to avoid modifying the default + configuration. + Changing the settings could result in unexpected results if used incorrectly. ```console kubectl create -f csi-ceph-conf-override.yaml @@ -392,7 +379,7 @@ Two changes are necessary to the configuration to enable this capability: ### Use hostNetwork in the rook ceph cluster configuration -Enable the `hostNetwork` setting in the [Ceph Cluster CRD configuration](ceph-cluster-crd.md#samples). +Enable the `hostNetwork` setting in the [Ceph Cluster CRD configuration](../../CRDs/ceph-cluster-crd.md#samples). For example, ```yaml @@ -400,8 +387,9 @@ For example, provider: host ``` -> IMPORTANT: Changing this setting is not supported in a running Rook cluster. Host networking -> should be configured when the cluster is first created. +!!! important + Changing this setting is not supported in a running Rook cluster. Host networking + should be configured when the cluster is first created. ### Define the subnets to use for public and private OSD networks @@ -434,32 +422,27 @@ between each restart to ensure the cluster goes back to "active/clean" state. ## Phantom OSD Removal If you have OSDs in which are not showing any disks, you can remove those "Phantom OSDs" by following the instructions below. -To check for "Phantom OSDs", you can run: +To check for "Phantom OSDs", you can run (example output): ```console -ceph osd tree +$ ceph osd tree +ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF +-1 57.38062 root default +-13 7.17258 host node1.example.com +2 hdd 3.61859 osd.2 up 1.00000 1.00000 +-7 0 host node2.example.com down 0 1.00000 ``` -An example output looks like this: - ->``` ->ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF ->-1 57.38062 root default ->-13 7.17258 host node1.example.com ->2 hdd 3.61859 osd.2 up 1.00000 1.00000 ->-7 0 host node2.example.com down 0 1.00000 ->``` - The host `node2.example.com` in the output has no disks, so it is most likely a "Phantom OSD". Now to remove it, use the ID in the first column of the output and replace `` with it. In the example output above the ID would be `-7`. The commands are: ```console -$ ceph osd out -$ ceph osd crush remove osd. -$ ceph auth del osd. -$ ceph osd rm +ceph osd out +ceph osd crush remove osd. +ceph auth del osd. +ceph osd rm ``` To recheck that the Phantom OSD was removed, re-run the following command and check if the OSD with the ID doesn't show up anymore: @@ -470,23 +453,27 @@ ceph osd tree ## Auto Expansion of OSDs -### Prerequisites +### Prerequisites for Auto Expansion of OSDs -1) A [PVC-based cluster](ceph-cluster-crd.md#pvc-based-cluster) deployed in dynamic provisioning environment with a `storageClassDeviceSet`. +1) A [PVC-based cluster](../../CRDs/ceph-cluster-crd.md#pvc-based-cluster) deployed in dynamic provisioning environment with a `storageClassDeviceSet`. -2) Create the Rook [Toolbox](ceph-toolbox.md). +2) Create the Rook [Toolbox](../../Troubleshooting/ceph-toolbox.md). ->Note: [Prometheus Operator](ceph-monitoring.md#prometheus-operator) and [Prometheus Instances](ceph-monitoring.md#prometheus-instances) are Prerequisites that are created by the auto-grow-storage script. +!!! note + [Prometheus Operator](../Monitoring/ceph-monitoring.md#prometheus-operator) and [Prometheus ../Monitoring/ceph-monitoring.mdnitoring.md#prometheus-instances) are Prerequisites that are created by the auto-grow-storage script. ### To scale OSDs Vertically Run the following script to auto-grow the size of OSDs on a PVC-based Rook-Ceph cluster whenever the OSDs have reached the storage near-full threshold. + ```console tests/scripts/auto-grow-storage.sh size --max maxSize --growth-rate percent ``` ->growth-rate percentage represents the percent increase you want in the OSD capacity and maxSize represent the maximum disk size. + +`growth-rate` percentage represents the percent increase you want in the OSD capacity and maxSize represent the maximum disk size. For example, if you need to increase the size of OSD by 30% and max disk size is 1Ti + ```console ./auto-grow-storage.sh size --max 1Ti --growth-rate 30 ``` @@ -494,12 +481,15 @@ For example, if you need to increase the size of OSD by 30% and max disk size is ### To scale OSDs Horizontally Run the following script to auto-grow the number of OSDs on a PVC-based Rook-Ceph cluster whenever the OSDs have reached the storage near-full threshold. + ```console tests/scripts/auto-grow-storage.sh count --max maxCount --count rate ``` ->Count of OSD represents the number of OSDs you need to add and maxCount represents the number of disks a storage cluster will support. + +Count of OSD represents the number of OSDs you need to add and maxCount represents the number of disks a storage cluster will support. For example, if you need to increase the number of OSDs by 3 and maxCount is 10 + ```console ./auto-grow-storage.sh count --max 10 --count 3 ``` diff --git a/Documentation/ceph-mon-health.md b/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md similarity index 84% rename from Documentation/ceph-mon-health.md rename to Documentation/Storage-Configuration/Advanced/ceph-mon-health.md index 0ae91a30b285..a220c8b1f3b7 100644 --- a/Documentation/ceph-mon-health.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md @@ -1,11 +1,7 @@ --- title: Monitor Health -weight: 11130 -indent: true --- -# Monitor Health - Failure in a distributed system is to be expected. Ceph was designed from the ground up to deal with the failures of a distributed system. At the next layer, Rook was designed from the ground up to automate recovery of Ceph components that traditionally required admin intervention. Monitor health is the most critical piece of the equation that Rook actively monitors. If they are not in a good state, @@ -36,7 +32,7 @@ You would still have 2/3 mons running to give you consensus in the cluster for a For highest availability, an odd number of mons is required. Fifty percent of mons will not be sufficient to maintain quorum. If you had two mons and one of them went down, you would have 1/2 of quorum. Since that is not a super-majority, the cluster would have to wait until the second mon is up again. -Rook allows an even number of mons for higher durability. See the [disaster recovery guide](ceph-disaster-recovery.md#restoring-mon-quorum) if quorum is lost and to recover mon quorum from a single mon. +Rook allows an even number of mons for higher durability. See the [disaster recovery guide](../../Troubleshooting/disaster-recovery.md#restoring-mon-quorum) if quorum is lost and to recover mon quorum from a single mon. The number of mons to create in a cluster depends on your tolerance for losing a node. If you have 1 mon zero nodes can be lost to maintain quorum. With 3 mons one node can be lost, and with 5 mons two nodes can be lost. Because the Rook operator will automatically @@ -86,41 +82,36 @@ This is especially useful for planned maintenance. ### Example Failover Rook will create mons with pod names such as mon-a, mon-b, and mon-c. Let's say mon-b had an issue and the pod failed. + ```console -kubectl -n rook-ceph get pod -l app=rook-ceph-mon +$ kubectl -n rook-ceph get pod -l app=rook-ceph-mon +NAME READY STATUS RESTARTS AGE +rook-ceph-mon-a-74dc96545-ch5ns 1/1 Running 0 9m +rook-ceph-mon-b-6b9d895c4c-bcl2h 1/1 Error 2 9m +rook-ceph-mon-c-7d6df6d65c-5cjwl 1/1 Running 0 8m ``` ->``` ->NAME READY STATUS RESTARTS AGE ->rook-ceph-mon-a-74dc96545-ch5ns 1/1 Running 0 9m ->rook-ceph-mon-b-6b9d895c4c-bcl2h 1/1 Error 2 9m ->rook-ceph-mon-c-7d6df6d65c-5cjwl 1/1 Running 0 8m ->``` - After a failover, you will see the unhealthy mon removed and a new mon added such as mon-d. A fully healthy mon quorum is now running again. + ```console -kubectl -n rook-ceph get pod -l app=rook-ceph-mon +$ kubectl -n rook-ceph get pod -l app=rook-ceph-mon +NAME READY STATUS RESTARTS AGE +rook-ceph-mon-a-74dc96545-ch5ns 1/1 Running 0 19m +rook-ceph-mon-c-7d6df6d65c-5cjwl 1/1 Running 0 18m +rook-ceph-mon-d-9e7ea7e76d-4bhxm 1/1 Running 0 20s ``` ->``` ->NAME READY STATUS RESTARTS AGE ->rook-ceph-mon-a-74dc96545-ch5ns 1/1 Running 0 19m ->rook-ceph-mon-c-7d6df6d65c-5cjwl 1/1 Running 0 18m ->rook-ceph-mon-d-9e7ea7e76d-4bhxm 1/1 Running 0 20s ->``` From the toolbox we can verify the status of the health mon quorum: + ```console -ceph -s +$ ceph -s + cluster: + id: 35179270-8a39-4e08-a352-a10c52bb04ff + health: HEALTH_OK + + services: + mon: 3 daemons, quorum a,b,d (age 2m) + mgr: a(active, since 12m) + osd: 3 osds: 3 up (since 10m), 3 in (since 10m) +[...] ``` - ->``` -> cluster: -> id: 35179270-8a39-4e08-a352-a10c52bb04ff -> health: HEALTH_OK -> -> services: -> mon: 3 daemons, quorum a,b,d (age 2m) -> mgr: a(active, since 12m) -> osd: 3 osds: 3 up (since 10m), 3 in (since 10m) -> ... ->``` diff --git a/Documentation/ceph-osd-mgmt.md b/Documentation/Storage-Configuration/Advanced/ceph-osd-mgmt.md similarity index 66% rename from Documentation/ceph-osd-mgmt.md rename to Documentation/Storage-Configuration/Advanced/ceph-osd-mgmt.md index 133eb96e84f7..c5fdeaa9310f 100644 --- a/Documentation/ceph-osd-mgmt.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-osd-mgmt.md @@ -1,11 +1,6 @@ --- -title: OSD Management -weight: 11140 -indent: true +title: Ceph OSD Management --- -{% include_relative branch.liquid %} - -# Ceph OSD Management Ceph Object Storage Daemons (OSDs) are the heart and soul of the Ceph storage platform. Each OSD manages a local device and together they provide the distributed storage. Rook will automate creation and management of OSDs to hide the complexity @@ -14,12 +9,13 @@ to configure OSDs where more configuration may be required. ## OSD Health -The [rook-ceph-tools pod](./ceph-toolbox.md) provides a simple environment to run Ceph tools. The `ceph` commands +The [rook-ceph-tools pod](../../Troubleshooting/ceph-toolbox.md) provides a simple environment to run Ceph tools. The `ceph` commands mentioned in this document should be run from the toolbox. Once the is created, connect to the pod to execute the `ceph` commands to analyze the health of the cluster, in particular the OSDs and placement groups (PGs). Some common commands to analyze OSDs include: -``` + +```console ceph status ceph osd tree ceph osd status @@ -33,8 +29,8 @@ kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-t ## Add an OSD -The [QuickStart Guide](quickstart.md) will provide the basic steps to create a cluster and start some OSDs. For more details on the OSD -settings also see the [Cluster CRD](ceph-cluster-crd.md) documentation. If you are not seeing OSDs created, see the [Ceph Troubleshooting Guide](ceph-common-issues.md). +The [QuickStart Guide](../../Getting-Started/quickstart.md) will provide the basic steps to create a cluster and start some OSDs. For more details on the OSD +settings also see the [Cluster CRD](../../CRDs/ceph-cluster-crd.md) documentation. If you are not seeing OSDs created, see the [Ceph Troubleshooting Guide](../../Troubleshooting/ceph-common-issues.md). To add more OSDs, Rook will automatically watch for new nodes and devices being added to your cluster. If they match the filters or other settings in the `storage` section of the cluster CR, the operator @@ -43,7 +39,7 @@ will create new OSDs. ## Add an OSD on a PVC In more dynamic environments where storage can be dynamically provisioned with a raw block storage provider, the OSDs can be backed -by PVCs. See the `storageClassDeviceSets` documentation in the [Cluster CRD](ceph-cluster-crd.md#storage-class-device-sets) topic. +by PVCs. See the `storageClassDeviceSets` documentation in the [Cluster CRD](../../CRDs/ceph-cluster-crd.md#storage-class-device-sets) topic. To add more OSDs, you can either increase the `count` of the OSDs in an existing device set or you can add more device sets to the cluster CR. The operator will then automatically create new OSDs according @@ -54,10 +50,10 @@ to the updated cluster CR. To remove an OSD due to a failed disk or other re-configuration, consider the following to ensure the health of the data through the removal process: -- Confirm you will have enough space on your cluster after removing your OSDs to properly handle the deletion -- Confirm the remaining OSDs and their placement groups (PGs) are healthy in order to handle the rebalancing of the data -- Do not remove too many OSDs at once -- Wait for rebalancing between removing multiple OSDs +* Confirm you will have enough space on your cluster after removing your OSDs to properly handle the deletion +* Confirm the remaining OSDs and their placement groups (PGs) are healthy in order to handle the rebalancing of the data +* Do not remove too many OSDs at once +* Wait for rebalancing between removing multiple OSDs If all the PGs are `active+clean` and there are no warnings about being low on space, this means the data is fully replicated and it is safe to proceed. If an OSD is failing, the PGs will not be perfectly clean and you will need to proceed anyway. @@ -67,34 +63,40 @@ and it is safe to proceed. If an OSD is failing, the PGs will not be perfectly c Update your CephCluster CR. Depending on your CR settings, you may need to remove the device from the list or update the device filter. If you are using `useAllDevices: true`, no change to the CR is necessary. -**IMPORTANT: On host-based clusters, you may need to stop the Rook Operator while performing OSD -removal steps in order to prevent Rook from detecting the old OSD and trying to re-create it before -the disk is wiped or removed.** +!!! important + **On host-based clusters, you may need to stop the Rook Operator while performing OSD + removal steps in order to prevent Rook from detecting the old OSD and trying to re-create it before the disk is wiped or removed.** -To stop the Rook Operator, run -`kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas=0`. +To stop the Rook Operator, run: + +```console +kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas=0 +``` You must perform steps below to (1) purge the OSD and either (2.a) delete the underlying data or (2.b)replace the disk before starting the Rook Operator again. -Once you have done that, you can start the Rook operator again with -`kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas=1`. +Once you have done that, you can start the Rook operator again with: + +```console +kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas=1 +``` ### PVC-based cluster To reduce the storage in your cluster or remove a failed OSD on a PVC: 1. Shrink the number of OSDs in the `storageClassDeviceSets` in the CephCluster CR. If you have multiple device sets, - you may need to change the index of `0` in this example path. - - `kubectl -n rook-ceph patch CephCluster rook-ceph --type=json -p '[{"op": "replace", "path": "/spec/storage/storageClassDeviceSets/0/count", "value":}]'` - - Reduce the `count` of the OSDs to the desired number. Rook will not take any action to automatically remove the extra OSD(s). + you may need to change the index of `0` in this example path. + * `kubectl -n rook-ceph patch CephCluster rook-ceph --type=json -p '[{"op": "replace", "path": "/spec/storage/storageClassDeviceSets/0/count", "value":}]'` + * Reduce the `count` of the OSDs to the desired number. Rook will not take any action to automatically remove the extra OSD(s). 2. Identify the PVC that belongs to the OSD that is failed or otherwise being removed. - - `kubectl -n rook-ceph get pvc -l ceph.rook.io/DeviceSet=` + * `kubectl -n rook-ceph get pvc -l ceph.rook.io/DeviceSet=` 3. Identify the OSD you desire to remove. - - The OSD assigned to the PVC can be found in the labels on the PVC - - `kubectl -n rook-ceph get pod -l ceph.rook.io/pvc= -o yaml | grep ceph-osd-id` - - For example, this might return: `ceph-osd-id: "0"` - - Remember the OSD ID for purging the OSD below + * The OSD assigned to the PVC can be found in the labels on the PVC + * `kubectl -n rook-ceph get pod -l ceph.rook.io/pvc= -o yaml | grep ceph-osd-id` + * For example, this might return: `ceph-osd-id: "0"` + * Remember the OSD ID for purging the OSD below If you later increase the count in the device set, note that the operator will create PVCs with the highest index that is not currently in use by existing OSD PVCs. @@ -102,12 +104,17 @@ that is not currently in use by existing OSD PVCs. ### Confirm the OSD is down If you want to remove an unhealthy OSD, the osd pod may be in an error state such as `CrashLoopBackoff` or the `ceph` commands -in the toolbox may show which OSD is `down`. If you want to remove a healthy OSD, you should run -`kubectl -n rook-ceph scale deployment rook-ceph-osd- --replicas=0` and `ceph osd down osd.` from the toolbox. +in the toolbox may show which OSD is `down`. If you want to remove a healthy OSD, you should run the following commands: + +```console +$ kubectl -n rook-ceph scale deployment rook-ceph-osd- --replicas=0 +# Inside the toolbox +$ ceph osd down osd. +``` ### Purge the OSD from the Ceph cluster -OSD removal can be automated with the example found in the [rook-ceph-purge-osd job](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/osd-purge.yaml). +OSD removal can be automated with the example found in the [rook-ceph-purge-osd job](https://github.com/rook/rook/blob/master/deploy/examples/osd-purge.yaml). In the osd-purge.yaml, change the `` to the ID(s) of the OSDs you want to remove. 1. Run the job: `kubectl create -f osd-purge.yaml` @@ -121,32 +128,36 @@ If you want to remove OSDs by hand, continue with the following sections. Howeve If the OSD purge job fails or you need fine-grained control of the removal, here are the individual commands that can be run from the toolbox. 1. Detach the OSD PVC from Rook - - `kubectl -n rook-ceph label pvc ceph.rook.io/DeviceSetPVCId-` + * `kubectl -n rook-ceph label pvc ceph.rook.io/DeviceSetPVCId-` 2. Mark the OSD as `out` if not already marked as such by Ceph. This signals Ceph to start moving (backfilling) the data that was on that OSD to another OSD. - - `ceph osd out osd.` (for example if the OSD ID is 23 this would be `ceph osd out osd.23`) + * `ceph osd out osd.` (for example if the OSD ID is 23 this would be `ceph osd out osd.23`) 3. Wait for the data to finish backfilling to other OSDs. - - `ceph status` will indicate the backfilling is done when all of the PGs are `active+clean`. If desired, it's safe to remove the disk after that. + * `ceph status` will indicate the backfilling is done when all of the PGs are `active+clean`. If desired, it's safe to remove the disk after that. 4. Remove the OSD from the Ceph cluster - - `ceph osd purge --yes-i-really-mean-it` + * `ceph osd purge --yes-i-really-mean-it` 5. Verify the OSD is removed from the node in the CRUSH map - - `ceph osd tree` + * `ceph osd tree` The operator can automatically remove OSD deployments that are considered "safe-to-destroy" by Ceph. After the steps above, the OSD will be considered safe to remove since the data has all been moved to other OSDs. But this will only be done automatically by the operator if you have this setting in the cluster CR: -``` + +```yaml removeOSDsIfOutAndSafeToRemove: true ``` Otherwise, you will need to delete the deployment directly: - - `kubectl delete deployment -n rook-ceph rook-ceph-osd-` + +```console +kubectl delete deployment -n rook-ceph rook-ceph-osd- +``` In PVC-based cluster, remove the orphaned PVC, if necessary. ### Delete the underlying data If you want to clean the device where the OSD was running, see in the instructions to -wipe a disk on the [Cleaning up a Cluster](ceph-teardown.md#delete-the-data-on-hosts) topic. +wipe a disk on the [Cleaning up a Cluster](../ceph-teardown.md#delete-the-data-on-hosts) topic. ## Replace an OSD @@ -160,4 +171,5 @@ If your cluster CR lists individual devices or uses a device filter you may need If you don't see a new OSD automatically created, restart the operator (by deleting the operator pod) to trigger the OSD creation. 5. Verify if the OSD is created on the node by running `ceph osd tree` from the toolbox. -Note that the OSD might have a different ID than the previous OSD that was replaced. +!!! note + The OSD might have a different ID than the previous OSD that was replaced. diff --git a/Documentation/ceph-configuration.md b/Documentation/Storage-Configuration/Advanced/configuration.md similarity index 81% rename from Documentation/ceph-configuration.md rename to Documentation/Storage-Configuration/Advanced/configuration.md index c729c7cb5ceb..24e5d6e9f4bf 100644 --- a/Documentation/ceph-configuration.md +++ b/Documentation/Storage-Configuration/Advanced/configuration.md @@ -1,20 +1,17 @@ --- title: Configuration -weight: 3700 -indent: true --- -# Configuration - For most any Ceph cluster, the user will want to--and may need to--change some Ceph configurations. These changes often may be warranted in order to alter performance to meet SLAs or to update default data resiliency settings. -> **WARNING**: Modify Ceph settings carefully, and review the -> [Ceph configuration documentation](https://docs.ceph.com/docs/master/rados/configuration/) before -> making any changes. -> Changing the settings could result in unhealthy daemons or even data loss if -> used incorrectly. +!!! warning + Modify Ceph settings carefully, and review the + [Ceph configuration documentation](https://docs.ceph.com/docs/master/rados/configuration/) before + making any changes. + Changing the settings could result in unhealthy daemons or even data loss if + used incorrectly. ## Required configurations @@ -36,7 +33,7 @@ for more information about this module. In Octopus (v15.2.x) and newer, module `pg_autoscaler` is enabled by default without the above-mentioned setting. -To disable this module, in the [CephCluster CR](ceph-cluster-crd.md#mgr-settings): +To disable this module, in the [CephCluster CR](../../CRDs/ceph-cluster-crd.md#mgr-settings): ```yaml spec: @@ -60,12 +57,12 @@ enable the autoscaling for these existing pools, they must be configured from th The most recommended way of configuring Ceph is to set Ceph's configuration directly. The first method for doing so is to use Ceph's CLI from the Rook-Ceph toolbox pod. Using the toolbox pod is -detailed [here](ceph-toolbox.md). From the toolbox, the user can change Ceph configurations, enable +detailed [here](../../Troubleshooting/ceph-toolbox.md). From the toolbox, the user can change Ceph configurations, enable manager modules, create users and pools, and much more. ### Ceph Dashboard -The Ceph Dashboard, examined in more detail [here](ceph-dashboard.md), is another way of setting +The Ceph Dashboard, examined in more detail [here](../Monitoring/ceph-dashboard.md), is another way of setting some of Ceph's configuration directly. Configuration by the Ceph dashboard is recommended with the same priority as configuration via the Ceph CLI (above). @@ -76,4 +73,4 @@ set, and setting configs via dashboard requires at least one mgr to be available a small number of very advanced settings that aren't able to be modified easily via CLI or dashboard. The **least** recommended method for configuring Ceph is intended as a last-resort fallback in situations like these. This is covered in detail -[here](ceph-advanced-configuration.md#custom-cephconf-settings). +[here](ceph-configuration.md#custom-cephconf-settings). diff --git a/Documentation/ceph-kms.md b/Documentation/Storage-Configuration/Advanced/key-management-system.md similarity index 89% rename from Documentation/ceph-kms.md rename to Documentation/Storage-Configuration/Advanced/key-management-system.md index 8fc95375ca87..ad0c5d4fed5d 100644 --- a/Documentation/ceph-kms.md +++ b/Documentation/Storage-Configuration/Advanced/key-management-system.md @@ -1,11 +1,7 @@ --- title: Key Management System -weight: 3650 -indent: true --- -# Key Management System - Rook has the ability to encrypt OSDs of clusters running on PVC via the flag (`encrypted: true`) in your `storageClassDeviceSets` [template](#pvc-based-cluster). By default, the Key Encryption Keys (also known as Data Encryption Keys) are stored in a Kubernetes Secret. However, if a Key Management System exists Rook is capable of using it. @@ -20,7 +16,13 @@ The `security` section contains settings related to encryption of the cluster. Supported KMS providers: * [Vault](#vault) -* [IBM Key Protect](#ibm-kp) + * [Authentication methods](#authentication-methods) + * [Token-based authentication](#token-based-authentication) + * [Kubernetes-based authentication](#kubernetes-based-authentication) + * [General Vault configuration](#general-vault-configuration) + * [TLS configuration](#tls-configuration) +* [IBM Key Protect](#ibm-key-protect) + * [Configuration](#configuration) ## Vault @@ -32,8 +34,8 @@ Rook support two authentication methods: * [token-based](#token-based-authentication): a token is provided by the user and is stored in a Kubernetes Secret. It's used to authenticate the KMS by the Rook operator. This has several pitfalls such as: - * when the token expires it must be renewed, so the secret holding it must be updated - * no token automatic rotation + * when the token expires it must be renewed, so the secret holding it must be updated + * no token automatic rotation * [Kubernetes Service Account](#kubernetes-based-authentication) uses [Vault Kubernetes native authentication](https://www.vaultproject.io/docs/auth/kubernetes) mechanism and alleviate some of the limitations from the token authentication such as token automatic renewal. This method is generally recommended over the token-based authentication. @@ -87,7 +89,7 @@ security: In order to use the Kubernetes Service Account authentication method, the following must be run to properly configure Vault: -```sh +```console ROOK_NAMESPACE=rook-ceph ROOK_VAULT_SA=rook-vault-auth ROOK_SYSTEM_SA=rook-ceph-system @@ -150,7 +152,8 @@ security: VAULT_AUTH_KUBERNETES_ROLE: rook-ceph ``` -> Note: The `VAULT_ADDR` value above assumes that Vault is accessible within the cluster itself on the default port (8200). If running elsewhere, please update the URL accordingly. +!!! note + The `VAULT_ADDR` value above assumes that Vault is accessible within the cluster itself on the default port (8200). If running elsewhere, please update the URL accordingly. ### General Vault configuration @@ -168,20 +171,18 @@ capabilities = ["read"] You can write the policy like so and then create a token: ```console -vault policy write rook /tmp/rook.hcl -vault token create -policy=rook +$ vault policy write rook /tmp/rook.hcl +$ vault token create -policy=rook +Key Value +--- ----- +token s.FYzlo02cgnTehF8dXnAoq2Z9 +token_accessor oMo7sAXQKbYtxU4HtO8k3pko +token_duration 768h +token_renewable true +token_policies ["default" "rook"] +identity_policies [] +policies ["default" "rook"] ``` ->``` ->Key Value ->--- ----- ->token s.FYzlo02cgnTehF8dXnAoq2Z9 ->token_accessor oMo7sAXQKbYtxU4HtO8k3pko ->token_duration 768h ->token_renewable true ->token_policies ["default" "rook"] ->identity_policies [] ->policies ["default" "rook"] ->``` In the above example, Vault's secret backend path name is `rook`. It must be enabled with the following: diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/.pages b/Documentation/Storage-Configuration/Block-Storage-RBD/.pages new file mode 100644 index 000000000000..8edb39c250a6 --- /dev/null +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/.pages @@ -0,0 +1,5 @@ +title: Block Storage (RBD) +nav: + - block-storage.md + - rbd-mirroring.md + - ... diff --git a/Documentation/ceph-block.md b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md similarity index 67% rename from Documentation/ceph-block.md rename to Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md index a22508b198f6..c3ab4868f39f 100644 --- a/Documentation/ceph-block.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md @@ -1,25 +1,21 @@ --- -title: Block Storage -weight: 2100 -indent: true +title: Block Storage Overview --- -{% include_relative branch.liquid %} - -# Block Storage Block storage allows a single pod to mount storage. This guide shows how to create a simple, multi-tier web application on Kubernetes using persistent volumes enabled by Rook. ## Prerequisites -This guide assumes a Rook cluster as explained in the [Quickstart](quickstart.md). +This guide assumes a Rook cluster as explained in the [Quickstart](../../Getting-Started/quickstart.md). ## Provision Storage -Before Rook can provision storage, a [`StorageClass`](https://kubernetes.io/docs/concepts/storage/storage-classes) and [`CephBlockPool`](ceph-pool-crd.md) need to be created. This will allow Kubernetes to interoperate with Rook when provisioning persistent volumes. +Before Rook can provision storage, a [`StorageClass`](https://kubernetes.io/docs/concepts/storage/storage-classes) and [`CephBlockPool` CR](../../CRDs/Block-Storage/ceph-block-pool-crd.md) need to be created. This will allow Kubernetes to interoperate with Rook when provisioning persistent volumes. -> **NOTE**: This sample requires *at least 1 OSD per node*, with each OSD located on *3 different nodes*. +!!! note + This sample requires *at least 1 OSD per node*, with each OSD located on *3 different nodes*. -Each OSD must be located on a different node, because the [`failureDomain`](ceph-pool-crd.md#spec) is set to `host` and the `replicated.size` is set to `3`. +Each OSD must be located on a different node, because the [`failureDomain`](../../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `replicated.size` is set to `3`. Save this `StorageClass` definition as `storageclass.yaml`: @@ -98,7 +94,8 @@ Create the storage class. kubectl create -f deploy/examples/csi/rbd/storageclass.yaml ``` -> **NOTE**: As [specified by Kubernetes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain), when using the `Retain` reclaim policy, any Ceph RBD image that is backed by a `PersistentVolume` will continue to exist even after the `PersistentVolume` has been deleted. These Ceph RBD images will need to be cleaned up manually using `rbd rm`. +!!! note + As [specified by Kubernetes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain), when using the `Retain` reclaim policy, any Ceph RBD image that is backed by a `PersistentVolume` will continue to exist even after the `PersistentVolume` has been deleted. These Ceph RBD images will need to be cleaned up manually using `rbd rm`. ## Consume the storage: Wordpress sample @@ -117,21 +114,25 @@ Both of these apps create a block volume and mount it to their respective pod. Y ```console kubectl get pvc ``` ->``` ->NAME STATUS VOLUME CAPACITY ACCESSMODES AGE ->mysql-pv-claim Bound pvc-95402dbc-efc0-11e6-bc9a-0cc47a3459ee 20Gi RWO 1m ->wp-pv-claim Bound pvc-39e43169-efc1-11e6-bc9a-0cc47a3459ee 20Gi RWO 1m -``` + +!!! example "Example Output: `kubectl get pvc`" + ```console + NAME STATUS VOLUME CAPACITY ACCESSMODES AGE + mysql-pv-claim Bound pvc-95402dbc-efc0-11e6-bc9a-0cc47a3459ee 20Gi RWO 1m + wp-pv-claim Bound pvc-39e43169-efc1-11e6-bc9a-0cc47a3459ee 20Gi RWO 1m + ``` Once the wordpress and mysql pods are in the `Running` state, get the cluster IP of the wordpress app and enter it in your browser: ```console kubectl get svc wordpress ``` ->``` ->NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->wordpress 10.3.0.155 80:30841/TCP 2m ->``` + +!!! example "Example Output: `kubectl get svc wordpress`" + ```console + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + wordpress 10.3.0.155 80:30841/TCP 2m + ``` You should see the wordpress app running. @@ -141,17 +142,18 @@ If you are using Minikube, the Wordpress URL can be retrieved with this one-line echo http://$(minikube ip):$(kubectl get service wordpress -o jsonpath='{.spec.ports[0].nodePort}') ``` -> **NOTE**: When running in a vagrant environment, there will be no external IP address to reach wordpress with. You will only be able to reach wordpress via the `CLUSTER-IP` from inside the Kubernetes cluster. +!!! note + When running in a vagrant environment, there will be no external IP address to reach wordpress with. You will only be able to reach wordpress via the `CLUSTER-IP` from inside the Kubernetes cluster. ## Consume the storage: Toolbox -With the pool that was created above, we can also create a block image and mount it directly in a pod. See the [Direct Block Tools](direct-tools.md#block-storage-tools) topic for more details. +With the pool that was created above, we can also create a block image and mount it directly in a pod. See the [Direct Block Tools](../../Troubleshooting/direct-tools.md#block-storage-tools) topic for more details. ## Teardown To clean up all the artifacts created by the block demo: -``` +```console kubectl delete -f wordpress.yaml kubectl delete -f mysql.yaml kubectl delete -n rook-ceph cephblockpools.ceph.rook.io replicapool @@ -163,22 +165,25 @@ kubectl delete storageclass rook-ceph-block If you want to use erasure coded pool with RBD, your OSDs must use `bluestore` as their `storeType`. Additionally the nodes that are going to mount the erasure coded RBD block storage must have Linux kernel >= `4.11`. -**NOTE**: This example requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. +!!! attention + This example requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. -The OSDs must be located on different nodes, because the [`failureDomain`](ceph-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). +The OSDs must be located on different nodes, because the [`failureDomain`](../../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). To be able to use an erasure coded pool you need to create two pools (as seen below in the definitions): one erasure coded and one replicated. -> **NOTE**: This example requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. -The OSDs must be located on different nodes, because the [`failureDomain`](ceph-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). +!!! attention + This example requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. + +The OSDs must be located on different nodes, because the [`failureDomain`](../../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). ### Erasure Coded CSI Driver The erasure coded pool must be set as the `dataPool` parameter in -[`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/csi/rbd/storage-class-ec.yaml) It is used for the data of the RBD images. +[`storageclass-ec.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/csi/rbd/storage-class-ec.yaml) It is used for the data of the RBD images. ### Erasure Coded Flex Driver The erasure coded pool must be set as the `dataBlockPool` parameter in -[`storageclass-ec.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/flex/storage-class-ec.yaml). It is used for +[`storageclass-ec.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/flex/storage-class-ec.yaml). It is used for the data of the RBD images. diff --git a/Documentation/async-disaster-recovery.md b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback.md similarity index 76% rename from Documentation/async-disaster-recovery.md rename to Documentation/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback.md index 9577976b33c9..36f8388dd2b5 100644 --- a/Documentation/async-disaster-recovery.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback.md @@ -1,33 +1,22 @@ --- -title: Failover and Failback -weight: 3245 -indent: true +title: RBD Asynchronous DR Failover and Failback --- -# RBD Asynchronous DR Failover and Failback - -## Table of Contents - -* [Planned Migration and Disaster Recovery](#planned-migration-and-disaster-recovery) -* [Planned Migration](#planned-migration) - * [Relocation](#relocation) -* [Disaster Recovery](#disaster-recovery) - * [Failover](#failover-abrupt-shutdown) - * [Failback](#failback-post-disaster-recovery) - ## Planned Migration and Disaster Recovery Rook comes with the volume replication support, which allows users to perform disaster recovery and planned migration of clusters. The following document will help to track the procedure for failover and failback in case of a Disaster recovery or Planned migration use cases. -> **Note**: The document assumes that RBD Mirroring is set up between the peer clusters. -> For information on rbd mirroring and how to set it up using rook, please refer to -> the [rbd-mirroring guide](rbd-mirroring.md). +!!! note + The document assumes that RBD Mirroring is set up between the peer clusters. + For information on rbd mirroring and how to set it up using rook, please refer to + the [rbd-mirroring guide](rbd-mirroring.md). ## Planned Migration -> Use cases: Datacenter maintenance, technology refresh, disaster avoidance, etc. +!!! info + Use cases: Datacenter maintenance, technology refresh, disaster avoidance, etc. ### Relocation @@ -37,8 +26,9 @@ The Relocation operation is the process of switching production to a The image should now be made *primary* on the secondary cluster so that the access can be resumed there. -> :memo: Periodic or one-time backup of -> the application should be available for restore on the secondary site (cluster-2). +!!! note + :memo: Periodic or one-time backup of + the application should be available for restore on the secondary site (cluster-2). Follow the below steps for planned migration of workload from the primary cluster to the secondary cluster: @@ -65,16 +55,20 @@ Follow the below steps for planned migration of workload from the primary * Once the Image is marked as `primary`, the PVC is now ready to be used. Now, we can scale up the applications to use the PVC. ->:memo: **WARNING**: In Async Disaster recovery use case, we don't get -> the complete data. -> We will only get the crash-consistent data based on the snapshot interval time. +!!! warning + :memo: In Async Disaster recovery use case, we don't get + the complete data. + We will only get the crash-consistent data based on the snapshot interval time. ## Disaster Recovery -> Use cases: Natural disasters, Power failures, System failures, and crashes, etc. +!!! info + Use cases: Natural disasters, Power failures, System failures, and crashes, etc. + +!!! note + To effectively resume operations after a failover/relocation, + backup of the kubernetes artifacts like deployment, PVC, PV, etc need to be created beforehand by the admin; so that the application can be restored on the peer cluster. For more information, see [backup and restore](rbd-mirroring.md#backup-&-restore). -> **NOTE:** To effectively resume operations after a failover/relocation, -> backup of the kubernetes artifacts like deployment, PVC, PV, etc need to be created beforehand by the admin; so that the application can be restored on the peer cluster. For more information, see [backup and restore](rbd-mirroring.md#backup-&-restore). ### Failover (abrupt shutdown) In case of Disaster recovery, create VolumeReplication CR at the Secondary Site. diff --git a/Documentation/rbd-mirroring.md b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md similarity index 68% rename from Documentation/rbd-mirroring.md rename to Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md index b43da2d6d0dd..e274f3ca2c89 100644 --- a/Documentation/rbd-mirroring.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md @@ -1,10 +1,7 @@ --- title: RBD Mirroring -weight: 3242 -indent: true --- -# RBD Mirroring ## Disaster Recovery Disaster recovery (DR) is an organization's ability to react to and recover from an incident that negatively affects business operations. @@ -26,22 +23,9 @@ One of the solutions, to achieve the same, is [RBD mirroring](https://docs.ceph. manually created RBD image mirror-snapshots to replicate crash-consistent RBD images between clusters. -> **Note**: This document sheds light on rbd mirroring and how to set it up using rook. -> For steps on failover or failback scenarios - -## Table of Contents - -* [Create RBD Pools](#create-rbd-pools) -* [Bootstrap Peers](#bootstrap-peers) -* [Configure the RBDMirror Daemon](#configure-the-rbdmirror-daemon) -* [Add mirroring peer information to RBD pools](#add-mirroring-peer-information-to-rbd-pools) -* [Enable CSI Replication Sidecars](#enable-csi-replication-sidecars) -* [Volume Replication Custom Resources](#volume-replication-custom-resources) -* [Enable mirroring on a PVC](#enable-mirroring-on-a-pvc) - * [Creating a VolumeReplicationClass CR](#create-a-volume-replication-class-cr) - * [Creating a VolumeReplications CR](#create-a-volumereplication-cr) - * [Check VolumeReplication CR status](async-disaster-recovery.md#checking-replication-status) -* [Backup and Restore](#backup-&-restore) +!!! note + This document sheds light on rbd mirroring and how to set it up using rook. + For steps on failover or failback scenarios ## Create RBD Pools @@ -68,19 +52,21 @@ spec: mode: image ``` -```bash +```console kubectl create -f pool-mirrored.yaml ``` * Repeat the steps on the peer cluster. -> **NOTE:** Pool name across the cluster peers must be the same -> for RBD replication to function. +!!! note + Pool name across the cluster peers must be the same + for RBD replication to function. -See the [CephBlockPool documentation](ceph-pool-crd.md#mirroring) for more details. +See the [CephBlockPool documentation](../../CRDs/Block-Storage/ceph-block-pool-crd.md#mirroring) for more details. -> **Note:** It is also feasible to edit existing pools and -> enable them for replication. +!!! note + It is also feasible to edit existing pools and + enable them for replication. ## Bootstrap Peers @@ -92,7 +78,7 @@ The following steps enable bootstrapping peers to discover and * For Bootstrapping a peer cluster its bootstrap secret is required. To determine the name of the secret that contains the bootstrap secret execute the following command on the remote cluster (cluster-2) -```bash +```console [cluster-2]$ kubectl get cephblockpool.ceph.rook.io/mirroredpool -n rook-ceph -ojsonpath='{.status.info.rbdMirrorBootstrapPeerSecretName}' ``` @@ -100,17 +86,14 @@ Here, `pool-peer-token-mirroredpool` is the desired bootstrap secret name. * The secret pool-peer-token-mirroredpool contains all the information related to the token and needs to be injected to the peer, to fetch the decoded secret: -```bash +```console [cluster-2]$ kubectl get secret -n rook-ceph pool-peer-token-mirroredpool -o jsonpath='{.data.token}'|base64 -d +eyJmc2lkIjoiNGQ1YmNiNDAtNDY3YS00OWVkLThjMGEtOWVhOGJkNDY2OTE3IiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDZ3hmZGdxN013R0JBQWZzcUtCaGpZVjJUZDRxVzJYQm5kemc9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMzkuMzY6MzMwMCx2MToxOTIuMTY4LjM5LjM2OjY3ODldIn0= ``` -> ```bash ->eyJmc2lkIjoiNGQ1YmNiNDAtNDY3YS00OWVkLThjMGEtOWVhOGJkNDY2OTE3IiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDZ3hmZGdxN013R0JBQWZzcUtCaGpZVjJUZDRxVzJYQm5kemc9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMzkuMzY6MzMwMCx2MToxOTIuMTY4LjM5LjM2OjY3ODldIn0= -> ``` - * With this Decoded value, create a secret on the primary site (cluster-1): -```bash +```console [cluster-1]$ kubectl -n rook-ceph create secret generic rbd-primary-site-secret --from-literal=token=eyJmc2lkIjoiNGQ1YmNiNDAtNDY3YS00OWVkLThjMGEtOWVhOGJkNDY2OTE3IiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDZ3hmZGdxN013R0JBQWZzcUtCaGpZVjJUZDRxVzJYQm5kemc9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMzkuMzY6MzMwMCx2MToxOTIuMTY4LjM5LjM2OjY3ODldIn0= --from-literal=pool=mirroredpool ``` @@ -144,53 +127,46 @@ spec: * Create the RBD mirror daemon -```bash +```console [cluster-1]$ kubectl create -f mirror.yaml -n rook-ceph ``` * Validate if `rbd-mirror` daemon pod is now up -```bash +```console [cluster-1]$ kubectl get pods -n rook-ceph +rook-ceph-rbd-mirror-a-6985b47c8c-dpv4k 1/1 Running 0 10s ``` -> ```bash -> rook-ceph-rbd-mirror-a-6985b47c8c-dpv4k 1/1 Running 0 10s -> ``` - * Verify that daemon health is OK -```bash +```console kubectl get cephblockpools.ceph.rook.io mirroredpool -n rook-ceph -o jsonpath='{.status.mirroringStatus.summary}' +{"daemon_health":"OK","health":"OK","image_health":"OK","states":{"replaying":1}} ``` -> ```bash -> {"daemon_health":"OK","health":"OK","image_health":"OK","states":{"replaying":1}} -> ``` - * Repeat the above steps on the peer cluster. - See the [CephRBDMirror CRD](ceph-rbd-mirror-crd.md) for more details on the mirroring settings. - +See the [CephRBDMirror CRD](../../CRDs/Block-Storage/ceph-rbd-mirror-crd.md) for more details on the mirroring settings. ## Add mirroring peer information to RBD pools Each pool can have its own peer. To add the peer information, patch the already created mirroring enabled pool to update the CephBlockPool CRD. -```bash +```console [cluster-1]$ kubectl -n rook-ceph patch cephblockpool mirroredpool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["rbd-primary-site-secret"]}}}}' ``` + ## Create VolumeReplication CRDs Volume Replication Operator follows controller pattern and provides extended APIs for storage disaster recovery. The extended APIs are provided via Custom Resource Definition(CRD). Create the VolumeReplication CRDs on all the peer clusters. -```bash -$ kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml - -$ kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml +```console +kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml +kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml ``` ## Enable CSI Replication Sidecars @@ -217,7 +193,7 @@ Execute the following steps on each peer cluster to enable the * Edit the `rook-ceph-operator-config` configmap and add the following configurations -```bash +```console kubectl edit cm rook-ceph-operator-config -n rook-ceph ``` @@ -244,48 +220,47 @@ the storage admin information required for the volume replication operator. * **VolumeReplication**: *VolumeReplication* is a namespaced resource that contains references to storage object to be replicated and VolumeReplicationClass corresponding to the driver providing replication. -> For more information, please refer to the -> [volume-replication-operator](https://github.com/csi-addons/volume-replication-operator). +!!! info + For more information, please refer to the + [volume-replication-operator](https://github.com/csi-addons/volume-replication-operator). ## Enable mirroring on a PVC Below guide assumes that we have a PVC (rbd-pvc) in BOUND state; created using *StorageClass* with `Retain` reclaimPolicy. -```bash +```console [cluster-1]$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +rbd-pvc Bound pvc-65dc0aac-5e15-4474-90f4-7a3532c621ec 1Gi RWO csi-rbd-sc 44s ``` -> -> ```bash -> NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -> rbd-pvc Bound pvc-65dc0aac-5e15-4474-90f4-7a3532c621ec 1Gi RWO csi-rbd-sc 44s -> ``` - ### Create a Volume Replication Class CR In this case, we create a Volume Replication Class on cluster-1 -```bash +```console [cluster-1]$ kubectl apply -f deploy/examples/volume-replication-class.yaml ``` -> **Note:** The `schedulingInterval` can be specified in formats of -> minutes, hours or days using suffix `m`,`h` and `d` respectively. -> The optional schedulingStartTime can be specified using the ISO 8601 -> time format. +!!! note + The `schedulingInterval` can be specified in formats of + minutes, hours or days using suffix `m`, `h` and `d` respectively. + The optional schedulingStartTime can be specified using the ISO 8601 + time format. ### Create a VolumeReplication CR * Once VolumeReplicationClass is created, create a Volume Replication for the PVC which we intend to replicate to secondary cluster. -```bash +```console [cluster-1]$ kubectl apply -f deploy/examples/volume-replication.yaml ``` ->:memo: *VolumeReplication* is a namespace scoped object. Thus, -> it should be created in the same namespace as of PVC. +!!! note + :memo: `VolumeReplication` is a namespace scoped object. Thus, + it should be created in the same namespace as of PVC. ### Checking Replication Status @@ -298,87 +273,86 @@ In this case, we create a Volume Replication Class on cluster-1 To check VolumeReplication CR status: -```bash +```console [cluster-1]$kubectl get volumereplication pvc-volumereplication -oyaml ``` ->```yaml ->... ->spec: -> dataSource: -> apiGroup: "" -> kind: PersistentVolumeClaim -> name: rbd-pvc -> replicationState: primary -> volumeReplicationClass: rbd-volumereplicationclass ->status: -> conditions: -> - lastTransitionTime: "2021-05-04T07:39:00Z" -> message: "" -> observedGeneration: 1 -> reason: Promoted -> status: "True" -> type: Completed -> - lastTransitionTime: "2021-05-04T07:39:00Z" -> message: "" -> observedGeneration: 1 -> reason: Healthy -> status: "False" -> type: Degraded -> - lastTransitionTime: "2021-05-04T07:39:00Z" -> message: "" -> observedGeneration: 1 -> reason: NotResyncing -> status: "False" -> type: Resyncing -> lastCompletionTime: "2021-05-04T07:39:00Z" -> lastStartTime: "2021-05-04T07:38:59Z" -> message: volume is marked primary -> observedGeneration: 1 -> state: Primary ->``` +```yaml +... +spec: + dataSource: + apiGroup: "" + kind: PersistentVolumeClaim + name: rbd-pvc + replicationState: primary + volumeReplicationClass: rbd-volumereplicationclass +status: + conditions: + - lastTransitionTime: "2021-05-04T07:39:00Z" + message: "" + observedGeneration: 1 + reason: Promoted + status: "True" + type: Completed + - lastTransitionTime: "2021-05-04T07:39:00Z" + message: "" + observedGeneration: 1 + reason: Healthy + status: "False" + type: Degraded + - lastTransitionTime: "2021-05-04T07:39:00Z" + message: "" + observedGeneration: 1 + reason: NotResyncing + status: "False" + type: Resyncing + lastCompletionTime: "2021-05-04T07:39:00Z" + lastStartTime: "2021-05-04T07:38:59Z" + message: volume is marked primary + observedGeneration: 1 + state: Primary +``` ## Backup & Restore -> **NOTE:** To effectively resume operations after a failover/relocation, -> backup of the kubernetes artifacts like deployment, PVC, PV, etc need to be created beforehand by the admin; so that the application can be restored on the peer cluster. +!!! note + To effectively resume operations after a failover/relocation, + backup of the kubernetes artifacts like deployment, PVC, PV, etc need to be created beforehand by the admin; so that the application can be restored on the peer cluster. Here, we take a backup of PVC and PV object on one site, so that they can be restored later to the peer cluster. -#### **Take backup on cluster-1** +### **Take backup on cluster-1** * Take backup of the PVC `rbd-pvc` -```bash +```console [cluster-1]$ kubectl get pvc rbd-pvc -oyaml > pvc-backup.yaml ``` * Take a backup of the PV, corresponding to the PVC -```bash +```console [cluster-1]$ kubectl get pv/pvc-65dc0aac-5e15-4474-90f4-7a3532c621ec -oyaml > pv_backup.yaml ``` -> **Note**: We can also take backup using external tools like **Velero**. -> See [velero documentation](https://velero.io/docs/main/) for more information. +!!! note + We can also take backup using external tools like **Velero**. + See [velero documentation](https://velero.io/docs/main/) for more information. #### **Restore the backup on cluster-2** * Create storageclass on the secondary cluster -```bash +```console [cluster-2]$ kubectl create -f examples/rbd/storageclass.yaml ``` * Create VolumeReplicationClass on the secondary cluster -```bash +```console [cluster-1]$ kubectl apply -f deploy/examples/volume-replication-class.yaml - ``` - -> ```bash -> volumereplicationclass.replication.storage.openshift.io/rbd-volumereplicationclass created -> ``` +volumereplicationclass.replication.storage.openshift.io/rbd-volumereplicationclass created +``` * If Persistent Volumes and Claims are created manually on the secondary cluster, remove the `claimRef` on the backed up PV objects in yaml files; so that the @@ -404,21 +378,18 @@ spec: * Apply the Persistent Volume backup from the primary cluster -```bash +```console [cluster-2]$ kubectl create -f pv-backup.yaml ``` * Apply the Persistent Volume claim from the restored backup -```bash +```console [cluster-2]$ kubectl create -f pvc-backup.yaml ``` -```bash +```console [cluster-2]$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +rbd-pvc Bound pvc-65dc0aac-5e15-4474-90f4-7a3532c621ec 1Gi RWO rook-ceph-block 44s ``` - -> ```bash -> NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -> rbd-pvc Bound pvc-65dc0aac-5e15-4474-90f4-7a3532c621ec 1Gi RWO rook-ceph-block 44s -> ``` diff --git a/Documentation/Storage-Configuration/Ceph-CSI/.pages b/Documentation/Storage-Configuration/Ceph-CSI/.pages new file mode 100644 index 000000000000..e1e1f9330c55 --- /dev/null +++ b/Documentation/Storage-Configuration/Ceph-CSI/.pages @@ -0,0 +1,3 @@ +nav: + - ceph-csi-drivers.md + - ... diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md similarity index 81% rename from Documentation/ceph-csi-drivers.md rename to Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md index 47ce98943df2..ae8a260667a1 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md @@ -1,26 +1,23 @@ --- -title: Ceph CSI -weight: 3200 -indent: true +title: Ceph CSI Drivers --- -{% include_relative branch.liquid %} -# Ceph CSI Drivers There are three CSI drivers integrated with Rook that will enable different scenarios: * RBD: This block storage driver is optimized for RWO pod access where only one pod may access the - storage. [More information](ceph-block.md). + storage. [More information](../Block-Storage-RBD/block-storage.md). * CephFS: This file storage driver allows for RWX with one or more pods accessing the same storage. - [More information](ceph-filesystem.md). + [More information](../Shared-Filesystem-CephFS/filesystem-storage.md). * NFS (experimental): This file storage driver allows creating NFS exports that can be mounted to pods, or the exports can be mounted directly via an NFS client from inside or outside the - Kubernetes cluster. [More information](ceph-nfs-crd.md#ceph-csi-nfs-provisioner-and-nfs-csi-driver). + Kubernetes cluster. [More information](../../CRDs/ceph-nfs-crd.md#ceph-csi-nfs-provisioner-and-nfs-csi-driver). The Ceph Filesysetem (CephFS) and RADOS Block Device (RBD) drivers are enabled automatically with the Rook operator. The NFS driver is disabled by default. All drivers will be started in the same namespace as the operator when the first CephCluster CR is created. ## Supported Versions + The supported Ceph CSI version is 3.3.0 or greater with Rook. Refer to ceph csi [releases](https://github.com/ceph/ceph-csi/releases) for more information. @@ -44,16 +41,13 @@ for example `curl -X get http://[pod ip]:[liveness-port][liveness-path] 2>/dev/ the expected output should be ```console -curl -X GET http://10.109.65.142:9080/metrics 2>/dev/null | grep csi +$ curl -X GET http://10.109.65.142:9080/metrics 2>/dev/null | grep csi +# HELP csi_liveness Liveness Probe +# TYPE csi_liveness gauge +csi_liveness 1 ``` ->``` -># HELP csi_liveness Liveness Probe -># TYPE csi_liveness gauge ->csi_liveness 1 ->``` - -Check the [monitoring doc](ceph-monitoring.md) to see how to integrate CSI +Check the [monitoring doc](../Monitoring/ceph-monitoring.md) to see how to integrate CSI liveness and grpc metrics into ceph monitoring. ## Dynamically Expand Volume @@ -89,14 +83,14 @@ Kubernetes version 1.21 or greater is required. 1. Install the volume replication CRDs: -```console -kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml -kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml -``` + ```console + kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplications.yaml + kubectl create -f https://raw.githubusercontent.com/csi-addons/volume-replication-operator/v0.3.0/config/crd/bases/replication.storage.openshift.io_volumereplicationclasses.yaml + ``` 2. Enable the volume replication controller: - - For Helm deployments see the [csi.volumeReplication.enabled setting](helm-operator.md#configuration). - - For non-Helm deployments set `CSI_ENABLE_VOLUME_REPLICATION: "true"` in operator.yaml + * For Helm deployments see the [`csi.volumeReplication.enabled` setting](../../Helm-Charts/operator-chart.md#configuration). + * For non-Helm deployments set `CSI_ENABLE_VOLUME_REPLICATION: "true"` in operator.yaml ## Ephemeral volume support @@ -127,7 +121,7 @@ provisioned and used by the pod with its lifecycle. The volumes are provisioned when pod get spawned and destroyed at time of pod delete. Refer to [ephemeral-doc](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes) for more info. -Also, See the example manifests for an [RBD ephemeral volume](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/pod-ephemeral.yaml) and a [CephFS ephemeral volume](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/pod-ephemeral.yaml). +Also, See the example manifests for an [RBD ephemeral volume](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/pod-ephemeral.yaml) and a [CephFS ephemeral volume](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/pod-ephemeral.yaml). ## CSI-Addons Controller @@ -137,7 +131,7 @@ The CSI-Addons Controller handles the requests from users to initiate an operati Users can deploy the controller by running the following commands: -```bash +```console kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.3.0/deploy/controller/crds.yaml kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.3.0/deploy/controller/rbac.yaml kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-addons/v0.3.0/deploy/controller/setup-controller.yaml @@ -157,25 +151,26 @@ sidecar: * Update the `rook-ceph-operator-config` configmap and patch the following configurations -```bash +```console kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_CSIADDONS": "true"' ``` * After enabling `CSI_ENABLE_CSIADDONS` in the configmap, a new sidecar container with name `csi-addons` should now start automatically in the RBD CSI provisioner and nodeplugin pods. -> NOTE: Make sure the version of ceph-csi used is v3.5.0+ +!!! note + Make sure the version of ceph-csi used is `v3.5.0+`. ### CSI-ADDONS Operation CSI-Addons supports the following operations: -- Reclaim Space - - [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#reclaimspacejob) - - [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#reclaimspacecronjob) - - [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#annotating-perstentvolumeclaims) -- Network Fencing - - [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/networkfence.md) +* Reclaim Space + * [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#reclaimspacejob) + * [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#reclaimspacecronjob) + * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/reclaimspace.md#annotating-perstentvolumeclaims) +* Network Fencing + * [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.3.0/docs/networkfence.md) ## Enable RBD Encryption Support @@ -184,7 +179,9 @@ Ceph-CSI supports encrypting individual RBD PersistentVolumeClaim with LUKS encr with full list of supported encryption configurations. A sample configmap can be found [here](https://github.com/ceph/ceph-csi/blob/v3.6.0/examples/kms/vault/kms-config.yaml). -> NOTE: Rook also supports OSD encryption (see `encryptedDevice` option [here](ceph-cluster-crd.md#osd-configuration-settings)). +!!! note + Rook also supports OSD encryption (see `encryptedDevice` option [here](../../CRDs/ceph-cluster-crd.md#osd-configuration-settings)). + Using both RBD PVC encryption and OSD encryption together will lead to double encryption and may reduce read/write performance. Unlike OSD encryption, existing ceph clusters can also enable Ceph-CSI RBD PVC encryption support and multiple kinds of encryption @@ -214,7 +211,7 @@ data: * Update the `rook-ceph-operator-config` configmap and patch the following configurations -```bash +```console kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_ENCRYPTION": "true"' ``` @@ -231,7 +228,7 @@ stringData: encryptionPassphrase: test-encryption ``` -* Create a new [storageclass](../deploy/examples/csi/rbd/storageclass.yaml) with additional parameters +* Create a new [storageclass](https://github.com/rook/rook/blob/master/deploy/examples/csi/rbd/storageclass.yaml) with additional parameters `encrypted: "true"` and `encryptionKMSID: ""`. An example is show below: ```yaml @@ -246,4 +243,4 @@ parameters: # ... ``` -* PVCs created using the new storageclass will be encrypted. \ No newline at end of file +* PVCs created using the new storageclass will be encrypted. diff --git a/Documentation/ceph-csi-snapshot.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md similarity index 54% rename from Documentation/ceph-csi-snapshot.md rename to Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md index cca4ca73a337..c66dd52e659a 100644 --- a/Documentation/ceph-csi-snapshot.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md @@ -1,9 +1,6 @@ --- title: Snapshots -weight: 3250 -indent: true --- -{% include_relative branch.liquid %} ## Prerequisites @@ -11,17 +8,19 @@ indent: true - Install the snapshot controller and snapshot v1/v1beta1 CRD as required. More info can be found [here](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0#usage). -Note: If only Alpha snapshots are available, enable snapshotter in `rook-ceph-operator-config` or helm chart `values.yaml`, change the external-snapshotter image to `k8s.gcr.io/sig-storage/csi-snapshotter:v1.2.2` and refer to the [alpha snapshots documentation](https://github.com/rook/rook/blob/release-1.3/Documentation/ceph-csi-drivers.md#rbd-snapshots) +!!! note + If only Alpha snapshots are available, enable snapshotter in `rook-ceph-operator-config` or helm chart `values.yaml`, change the external-snapshotter image to `k8s.gcr.io/sig-storage/csi-snapshotter:v1.2.2` and refer to the [alpha snapshots documentation](https://github.com/rook/rook/blob/release-1.3/Documentation/ceph-csi-drivers.md#rbd-snapshots) -* We also need a `VolumeSnapshotClass` for volume snapshot to work. The purpose of a `VolumeSnapshotClass` is +- We also need a `VolumeSnapshotClass` for volume snapshot to work. The purpose of a `VolumeSnapshotClass` is defined in [the kubernetes documentation](https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes/). In short, as the documentation describes it: -> Just like StorageClass provides a way for administrators to describe the -> "classes" of storage they offer when provisioning a volume, -> VolumeSnapshotClass provides a way to describe the "classes" of storage when -> provisioning a volume snapshot. +!!! info + Just like StorageClass provides a way for administrators to describe the + "classes" of storage they offer when provisioning a volume, + VolumeSnapshotClass provides a way to describe the "classes" of storage when + provisioning a volume snapshot. ## Upgrade Snapshot API @@ -31,7 +30,7 @@ If your Kubernetes version is updated to a newer version of the snapshot API, fo ### VolumeSnapshotClass -In [VolumeSnapshotClass](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/snapshotclass.yaml), +In [VolumeSnapshotClass](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/snapshotclass.yaml), the `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the name of the secret created for the rbdplugin and `pool` to reflect the Ceph pool name. @@ -46,7 +45,7 @@ kubectl create -f deploy/examples/csi/rbd/snapshotclass.yaml ### Volumesnapshot -In [snapshot](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/snapshot.yaml), +In [snapshot](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/snapshot.yaml), `volumeSnapshotClassName` should be the name of the `VolumeSnapshotClass` previously created. The `persistentVolumeClaimName` should be the name of the PVC which is already created by the RBD CSI driver. @@ -58,30 +57,23 @@ kubectl create -f deploy/examples/csi/rbd/snapshot.yaml ### Verify RBD Snapshot Creation ```console -kubectl get volumesnapshotclass +$ kubectl get volumesnapshotclass +NAME DRIVER DELETIONPOLICY AGE +csi-rbdplugin-snapclass rook-ceph.rbd.csi.ceph.com Delete 3h55m ``` ->``` ->NAME DRIVER DELETIONPOLICY AGE ->csi-rbdplugin-snapclass rook-ceph.rbd.csi.ceph.com Delete 3h55m ->``` - ```console -kubectl get volumesnapshot +$ kubectl get volumesnapshot +NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE +rbd-pvc-snapshot true rbd-pvc 1Gi csi-rbdplugin-snapclass snapcontent-79090db0-7c66-4b18-bf4a-634772c7cac7 3h50m 3h51m ``` ->``` ->NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE ->rbd-pvc-snapshot true rbd-pvc 1Gi csi-rbdplugin-snapclass snapcontent-79090db0-7c66-4b18-bf4a-634772c7cac7 3h50m 3h51m ->``` - The snapshot will be ready to restore to a new PVC when the `READYTOUSE` field of the `volumesnapshot` is set to true. ### Restore the snapshot to a new PVC -In -[pvc-restore](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/pvc-restore.yaml), +In [pvc-restore](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/pvc-restore.yaml), `dataSource` should be the name of the `VolumeSnapshot` previously created. The `dataSource` kind should be the `VolumeSnapshot`. @@ -94,13 +86,11 @@ kubectl create -f deploy/examples/csi/rbd/pvc-restore.yaml ### Verify RBD Clone PVC Creation ```console -kubectl get pvc +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +rbd-pvc Bound pvc-84294e34-577a-11e9-b34f-525400581048 1Gi RWO rook-ceph-block 34m +rbd-pvc-restore Bound pvc-575537bf-577f-11e9-b34f-525400581048 1Gi RWO rook-ceph-block 8s ``` ->``` ->NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ->rbd-pvc Bound pvc-84294e34-577a-11e9-b34f-525400581048 1Gi RWO rook-ceph-block 34m ->rbd-pvc-restore Bound pvc-575537bf-577f-11e9-b34f-525400581048 1Gi RWO rook-ceph-block 8s ->``` ## RBD snapshot resource Cleanup @@ -116,7 +106,7 @@ kubectl delete -f deploy/examples/csi/rbd/snapshotclass.yaml ### VolumeSnapshotClass -In [VolumeSnapshotClass](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/snapshotclass.yaml), +In [VolumeSnapshotClass](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/snapshotclass.yaml), the `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the name of the secret created for the cephfsplugin. @@ -125,14 +115,13 @@ running in. When Ceph CSI is deployed by Rook, the operator will automatically maintain a configmap whose contents will match this key. By default this is "rook-ceph". - ```console kubectl create -f deploy/examples/csi/cephfs/snapshotclass.yaml ``` -### Volumesnapshot +### VolumeSnapshot -In [snapshot](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/snapshot.yaml), +In [snapshot](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/snapshot.yaml), `volumeSnapshotClassName` should be the name of the `VolumeSnapshotClass` previously created. The `persistentVolumeClaimName` should be the name of the PVC which is already created by the CephFS CSI driver. @@ -144,28 +133,24 @@ kubectl create -f deploy/examples/csi/cephfs/snapshot.yaml ### Verify CephFS Snapshot Creation ```console -kubectl get volumesnapshotclass +$ kubectl get volumesnapshotclass +NAME DRIVER DELETIONPOLICY AGE +csi-cephfslugin-snapclass rook-ceph.cephfs.csi.ceph.com Delete 3h55m ``` ->``` ->NAME DRIVER DELETIONPOLICY AGE ->csi-cephfslugin-snapclass rook-ceph.cephfs.csi.ceph.com Delete 3h55m ->``` + ```console -kubectl get volumesnapshot +$ kubectl get volumesnapshot +NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE +cephfs-pvc-snapshot true cephfs-pvc 1Gi csi-cephfsplugin-snapclass snapcontent-34476204-a14a-4d59-bfbc-2bbba695652c 3h50m 3h51m ``` ->``` ->NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE ->cephfs-pvc-snapshot true cephfs-pvc 1Gi csi-cephfsplugin-snapclass snapcontent-34476204-a14a-4d59-bfbc-2bbba695652c 3h50m 3h51m ->``` - The snapshot will be ready to restore to a new PVC when `READYTOUSE` field of the `volumesnapshot` is set to true. ### Restore the snapshot to a new PVC In -[pvc-restore](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/pvc-restore.yaml), +[pvc-restore](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/pvc-restore.yaml), `dataSource` should be the name of the `VolumeSnapshot` previously created. The `dataSource` kind should be the `VolumeSnapshot`. @@ -178,15 +163,12 @@ kubectl create -f deploy/examples/csi/cephfs/pvc-restore.yaml ### Verify CephFS Restore PVC Creation ```console -kubectl get pvc +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +cephfs-pvc Bound pvc-74734901-577a-11e9-b34f-525400581048 1Gi RWX rook-cephfs 55m +cephfs-pvc-restore Bound pvc-95308c75-6c93-4928-a551-6b5137192209 1Gi RWX rook-cephfs 34s ``` ->``` ->NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ->cephfs-pvc Bound pvc-74734901-577a-11e9-b34f-525400581048 1Gi RWX rook-cephfs 55m ->cephfs-pvc-restore Bound pvc-95308c75-6c93-4928-a551-6b5137192209 1Gi RWX rook-cephfs 34s ->``` - ## CephFS snapshot resource Cleanup To clean your cluster of the resources created by this example, run the following: @@ -199,5 +181,5 @@ kubectl delete -f deploy/examples/csi/cephfs/snapshotclass.yaml ## Limitations -* There is a limit of 400 snapshots per cephFS filesystem. -* The PVC cannot be deleted if it has snapshots. make sure all the snapshots on the PVC are deleted before you delete the PVC. +- There is a limit of 400 snapshots per cephFS filesystem. +- The PVC cannot be deleted if it has snapshots. make sure all the snapshots on the PVC are deleted before you delete the PVC. diff --git a/Documentation/ceph-csi-volume-clone.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone.md similarity index 91% rename from Documentation/ceph-csi-volume-clone.md rename to Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone.md index 6cc626d2df9d..9e2d4f325e80 100644 --- a/Documentation/ceph-csi-volume-clone.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone.md @@ -1,9 +1,6 @@ --- title: Volume clone -weight: 3250 -indent: true --- -{% include_relative branch.liquid %} The CSI Volume Cloning feature adds support for specifying existing PVCs in the `dataSource` field to indicate a user would like to clone a Volume. @@ -26,7 +23,7 @@ for more info. ### Volume Cloning In -[pvc-clone](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/rbd/pvc-clone.yaml), +[pvc-clone](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/pvc-clone.yaml), `dataSource` should be the name of the `PVC` which is already created by RBD CSI driver. The `dataSource` kind should be the `PersistentVolumeClaim` and also storageclass should be same as the source `PVC`. @@ -67,7 +64,7 @@ kubectl delete -f deploy/examples/csi/rbd/pvc-clone.yaml ### Volume Cloning In -[pvc-clone](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples/csi/cephfs/pvc-clone.yaml), +[pvc-clone](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/pvc-clone.yaml), `dataSource` should be the name of the `PVC` which is already created by CephFS CSI driver. The `dataSource` kind should be the `PersistentVolumeClaim` and also storageclass should be same as the source `PVC`. diff --git a/Documentation/Storage-Configuration/Monitoring/.pages b/Documentation/Storage-Configuration/Monitoring/.pages new file mode 100644 index 000000000000..991922b3b2eb --- /dev/null +++ b/Documentation/Storage-Configuration/Monitoring/.pages @@ -0,0 +1,4 @@ +nav: + - ceph-dashboard.md + - ceph-monitoring.md + - ... diff --git a/Documentation/ceph-dashboard.md b/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md similarity index 75% rename from Documentation/ceph-dashboard.md rename to Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md index 42658217570d..188f719ce74a 100755 --- a/Documentation/ceph-dashboard.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md @@ -1,16 +1,12 @@ --- title: Ceph Dashboard -weight: 2400 -indent: true --- -# Ceph Dashboard - The dashboard is a very helpful tool to give you an overview of the status of your Ceph cluster, including overall health, status of the mon quorum, status of the mgr, osd, and other Ceph daemons, view pools and PG status, show logs for the daemons, and more. Rook makes it simple to enable the dashboard. -![The Ceph dashboard](media/ceph-dashboard.png) +![The Ceph dashboard](ceph-dashboard/ceph-dashboard.png) ## Enable the Ceph Dashboard @@ -18,9 +14,10 @@ The [dashboard](https://docs.ceph.com/en/latest/mgr/dashboard/) can be enabled w This is the default setting in the example manifests. ```yaml - spec: - dashboard: - enabled: true +[...] +spec: + dashboard: + enabled: true ``` The Rook operator will enable the ceph-mgr dashboard module. A service object will be created to expose that port inside the Kubernetes cluster. Rook will @@ -29,22 +26,17 @@ enable port 8443 for https access. This example shows that port 8443 was configured. ```console -kubectl -n rook-ceph get service +$ kubectl -n rook-ceph get service +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 3h +rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 8443/TCP 3h ``` ->``` ->NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 3h ->rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 8443/TCP 3h ->``` - The first service is for reporting the [Prometheus metrics](ceph-monitoring.md), while the latter service is for the dashboard. If you are on a node in the cluster, you will be able to connect to the dashboard by using either the DNS name of the service at `https://rook-ceph-mgr-dashboard-https:8443` or by connecting to the cluster IP, in this example at `https://10.110.113.240:8443`. -> **IMPORTANT:** Please note the dashboard will only be enabled for the first Ceph object store created by Rook. - ### Login Credentials After you connect to the dashboard you will need to login for secure access. Rook creates a default user named @@ -60,11 +52,11 @@ kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['dat The following dashboard configuration settings are supported: ```yaml - spec: - dashboard: - urlPrefix: /ceph-dashboard - port: 8443 - ssl: true +spec: + dashboard: + urlPrefix: /ceph-dashboard + port: 8443 + ssl: true ``` * `urlPrefix` If you are accessing the dashboard via a reverse proxy, you may @@ -122,16 +114,13 @@ kubectl create -f dashboard-external-https.yaml You will see the new service `rook-ceph-mgr-dashboard-external-https` created: ```console -kubectl -n rook-ceph get service +$ kubectl -n rook-ceph get service +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 4h +rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 8443/TCP 4h +rook-ceph-mgr-dashboard-external-https NodePort 10.101.209.6 8443:31176/TCP 4h ``` ->``` ->NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->rook-ceph-mgr ClusterIP 10.108.111.192 9283/TCP 4h ->rook-ceph-mgr-dashboard ClusterIP 10.110.113.240 8443/TCP 4h ->rook-ceph-mgr-dashboard-external-https NodePort 10.101.209.6 8443:31176/TCP 4h ->``` - In this example, port `31176` will be opened to expose port `8443` from the ceph-mgr pod. Find the ip address of the VM. If using minikube, you can run `minikube ip` to find the ip address. Now you can enter the URL in your browser such as `https://192.168.99.110:31176` and the dashboard will appear. @@ -157,16 +146,13 @@ kubectl create -f dashboard-loadbalancer.yaml You will see the new service `rook-ceph-mgr-dashboard-loadbalancer` created: ```console -kubectl -n rook-ceph get service +$ kubectl -n rook-ceph get service +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-mgr ClusterIP 172.30.11.40 9283/TCP 4h +rook-ceph-mgr-dashboard ClusterIP 172.30.203.185 8443/TCP 4h +rook-ceph-mgr-dashboard-loadbalancer LoadBalancer 172.30.27.242 a7f23e8e2839511e9b7a5122b08f2038-1251669398.us-east-1.elb.amazonaws.com 8443:32747/TCP 4h ``` ->``` ->NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->rook-ceph-mgr ClusterIP 172.30.11.40 9283/TCP 4h ->rook-ceph-mgr-dashboard ClusterIP 172.30.203.185 8443/TCP 4h ->rook-ceph-mgr-dashboard-loadbalancer LoadBalancer 172.30.27.242 a7f23e8e2839511e9b7a5122b08f2038-1251669398.us-east-1.elb.amazonaws.com 8443:32747/TCP 4h ->``` - Now you can enter the URL in your browser such as `https://a7f23e8e2839511e9b7a5122b08f2038-1251669398.us-east-1.elb.amazonaws.com:8443` and the dashboard will appear. ### Ingress Controller @@ -222,23 +208,17 @@ kubectl create -f dashboard-ingress-https.yaml You will see the new Ingress `rook-ceph-mgr-dashboard` created: ```console -kubectl -n rook-ceph get ingress +$ kubectl -n rook-ceph get ingress +NAME HOSTS ADDRESS PORTS AGE +rook-ceph-mgr-dashboard rook-ceph.example.com 80, 443 5m ``` ->``` ->NAME HOSTS ADDRESS PORTS AGE ->rook-ceph-mgr-dashboard rook-ceph.example.com 80, 443 5m ->``` - And the new Secret for the TLS certificate: ```console kubectl -n rook-ceph get secret rook-ceph.example.com +NAME TYPE DATA AGE +rook-ceph.example.com kubernetes.io/tls 2 4m ``` ->``` ->NAME TYPE DATA AGE ->rook-ceph.example.com kubernetes.io/tls 2 4m ->``` - You can now browse to `https://rook-ceph.example.com/` to log into the dashboard. diff --git a/Documentation/media/ceph-dashboard.png b/Documentation/Storage-Configuration/Monitoring/ceph-dashboard/ceph-dashboard.png similarity index 100% rename from Documentation/media/ceph-dashboard.png rename to Documentation/Storage-Configuration/Monitoring/ceph-dashboard/ceph-dashboard.png diff --git a/Documentation/ceph-monitoring.md b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md similarity index 78% rename from Documentation/ceph-monitoring.md rename to Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md index c04702cb6fd2..8edcd65615e4 100644 --- a/Documentation/ceph-monitoring.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md @@ -1,11 +1,6 @@ --- title: Prometheus Monitoring -weight: 2500 -indent: true --- -{% include_relative branch.liquid %} - -# Prometheus Monitoring Each Rook Ceph cluster has some built in metrics collectors/exporters for monitoring with [Prometheus](https://prometheus.io/). @@ -13,11 +8,12 @@ If you do not have Prometheus running, follow the steps below to enable monitori contains a Prometheus instance, it will automatically discover Rook's scrape endpoint using the standard `prometheus.io/scrape` and `prometheus.io/port` annotations. -> **NOTE**: This assumes that the Prometheus instances is searching all your Kubernetes namespaces for Pods with these annotations. -> If prometheus is already installed in a cluster, it may not be configured to watch for third-party service monitors such as for Rook. -> Normally you should be able to add the prometheus annotations "prometheus.io/scrape=true" and prometheus.io/port={port} and -> prometheus would automatically configure the scrape points and start gathering metrics. If prometheus isn't configured to do this, see the -> [prometheus operator docs](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#prometheusioscrape). +!!! attention + This assumes that the Prometheus instances is searching all your Kubernetes namespaces for Pods with these annotations. + If prometheus is already installed in a cluster, it may not be configured to watch for third-party service monitors such as for Rook. + Normally you should be able to add the prometheus annotations `prometheus.io/scrape=true` and `prometheus.io/port={port}` and + prometheus would automatically configure the scrape points and start gathering metrics. If prometheus isn't configured to do this, see the + [prometheus operator docs](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#prometheusioscrape). ## Prometheus Operator @@ -42,7 +38,7 @@ With the Prometheus operator running, we can create a service monitor that will From the root of your locally cloned Rook repo, go the monitoring directory: ```console -$ git clone --single-branch --branch {{ branchName }} https://github.com/rook/rook.git +$ git clone --single-branch --branch master https://github.com/rook/rook.git cd rook/deploy/examples/monitoring ``` @@ -60,8 +56,9 @@ Ensure that the Prometheus server pod gets created and advances to the `Running` kubectl -n rook-ceph get pod prometheus-rook-prometheus-0 ``` -> **NOTE**: It is not recommended to consume storage from the Ceph cluster for Prometheus. -> If the Ceph cluster fails, Prometheus would become unresponsive and thus not alert you of the failure. +!!! note + It is not recommended to consume storage from the Ceph cluster for Prometheus. + If the Ceph cluster fails, Prometheus would become unresponsive and thus not alert you of the failure. ## Prometheus Web Console @@ -73,23 +70,23 @@ echo "http://$(kubectl -n rook-ceph -o jsonpath={.status.hostIP} get pod prometh You should now see the Prometheus monitoring website. -![Prometheus Monitoring Website](media/prometheus-monitor.png) +![Prometheus Monitoring Website](ceph-monitoring/prometheus-monitor.png) Click on `Graph` in the top navigation bar. -![Prometheus Add graph](media/prometheus-graph.png) +![Prometheus Add graph](ceph-monitoring/prometheus-graph.png) In the dropdown that says `insert metric at cursor`, select any metric you would like to see, for example `ceph_cluster_total_used_bytes` -![Prometheus Select Metric](media/prometheus-metric-cursor.png) +![Prometheus Select Metric](ceph-monitoring/prometheus-metric-cursor.png) Click on the `Execute` button. -![Prometheus Execute Metric](media/prometheus-execute-metric-cursor.png) +![Prometheus Execute Metric](ceph-monitoring/prometheus-execute-metric-cursor.png) Below the `Execute` button, ensure the `Graph` tab is selected and you should now see a graph of your chosen metric over time. -![Prometheus Execute Metric](media/prometheus-metric-cursor-graph.png) +![Prometheus Execute Metric](ceph-monitoring/prometheus-metric-cursor-graph.png) ## Prometheus Consoles @@ -100,9 +97,10 @@ A guide to how you can write your own Prometheus consoles can be found on the of ## Prometheus Alerts To enable the Ceph Prometheus alerts via the helm charts, set the following properties in values.yaml: -- rook-ceph chart: + +* rook-ceph chart: `monitoring.enabled: true` -- rook-ceph-cluster chart: +* rook-ceph-cluster chart: `monitoring.enabled: true` `monitoring.createPrometheusRules: true` @@ -110,34 +108,35 @@ Alternatively, to enable the Ceph Prometheus alerts with example manifests follo 1. Create the RBAC and prometheus rules: -```console -kubectl create -f deploy/examples/monitoring/rbac.yaml -kubectl create -f deploy/examples/monitoring/localrules.yaml -``` + ```console + kubectl create -f deploy/examples/monitoring/rbac.yaml + kubectl create -f deploy/examples/monitoring/localrules.yaml + ``` 2. Make following changes to your CephCluster object (e.g., `cluster.yaml`). -```YAML -apiVersion: ceph.rook.io/v1 -kind: CephCluster -metadata: - name: rook-ceph - namespace: rook-ceph -[...] -spec: -[...] - monitoring: - enabled: true -[...] -``` + ```YAML + apiVersion: ceph.rook.io/v1 + kind: CephCluster + metadata: + name: rook-ceph + namespace: rook-ceph + [...] + spec: + [...] + monitoring: + enabled: true + [...] + ``` 3. Deploy or update the CephCluster object. -```console -kubectl apply -f cluster.yaml -``` + ```console + kubectl apply -f cluster.yaml + ``` -> **NOTE**: This expects the Prometheus Operator and a Prometheus instance to be pre-installed by the admin. +!!! note + This expects the Prometheus Operator and a Prometheus instance to be pre-installed by the admin. ### Customize Alerts @@ -189,9 +188,9 @@ kubectl create -f updated-chart.yaml The dashboards have been created by [@galexrt](https://github.com/galexrt). For feedback on the dashboards please reach out to him on the [Rook.io Slack](https://slack.rook.io). -> **NOTE**: The dashboards are only compatible with Grafana 7.2.0 or higher. -> -> Also note that the dashboards are updated from time to time, to fix issues and improve them. +!!! note + The dashboards are only compatible with Grafana 7.2.0 or higher. + Also note that the dashboards are updated from time to time, to fix issues and improve them. The following Grafana dashboards are available: @@ -209,7 +208,8 @@ like `common.yaml`. kubectl apply -f deploy/examples/monitoring/rbac.yaml ``` -> This is updated automatically if you are upgrading via the helm chart +!!! hint + This is updated automatically if you are upgrading via the helm chart ## Teardown @@ -271,6 +271,7 @@ spec: Using metrics exported from the Prometheus service, the horizontal pod scaling can use the custom metrics other than CPU and memory consumption. It can be done with help of Prometheus Scaler provided by the [KEDA](https://keda.sh/docs/2.5/scalers/prometheus/). See the [KEDA deployment guide](https://keda.sh/docs/2.5/deploy/) for details. Following is an example to autoscale RGW: + ```YAML apiVersion: keda.sh/v1alpha1 kind: ScaledObject diff --git a/Documentation/media/prometheus-execute-metric-cursor.png b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-execute-metric-cursor.png similarity index 100% rename from Documentation/media/prometheus-execute-metric-cursor.png rename to Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-execute-metric-cursor.png diff --git a/Documentation/media/prometheus-graph.png b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-graph.png similarity index 100% rename from Documentation/media/prometheus-graph.png rename to Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-graph.png diff --git a/Documentation/media/prometheus-metric-cursor-graph.png b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-metric-cursor-graph.png similarity index 100% rename from Documentation/media/prometheus-metric-cursor-graph.png rename to Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-metric-cursor-graph.png diff --git a/Documentation/media/prometheus-metric-cursor.png b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-metric-cursor.png similarity index 100% rename from Documentation/media/prometheus-metric-cursor.png rename to Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-metric-cursor.png diff --git a/Documentation/media/prometheus-monitor.png b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-monitor.png similarity index 100% rename from Documentation/media/prometheus-monitor.png rename to Documentation/Storage-Configuration/Monitoring/ceph-monitoring/prometheus-monitor.png diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/.pages b/Documentation/Storage-Configuration/Object-Storage-RGW/.pages new file mode 100644 index 000000000000..ab905e1a202c --- /dev/null +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/.pages @@ -0,0 +1,4 @@ +title: Object Storage (RGW) +nav: + - object-storage.md + - ... diff --git a/Documentation/ceph-object-bucket-claim.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md similarity index 88% rename from Documentation/ceph-object-bucket-claim.md rename to Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md index ae87b72be013..1e1a7149d943 100644 --- a/Documentation/ceph-object-bucket-claim.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md @@ -1,22 +1,19 @@ --- -title: Object Bucket Claim -weight: 2850 -indent: true +title: Bucket Claim --- -# Ceph Object Bucket Claim - Rook supports the creation of new buckets and access to existing buckets via two custom resources: -- an `Object Bucket Claim (OBC)` is custom resource which requests a bucket (new or existing) and is described by a Custom Resource Definition (CRD) shown below. -- an `Object Bucket (OB)` is a custom resource automatically generated when a bucket is provisioned. It is a global resource, typically not visible to non-admin users, and contains information specific to the bucket. It is described by an OB CRD, also shown below. +* an `Object Bucket Claim (OBC)` is custom resource which requests a bucket (new or existing) and is described by a Custom Resource Definition (CRD) shown below. +* an `Object Bucket (OB)` is a custom resource automatically generated when a bucket is provisioned. It is a global resource, typically not visible to non-admin users, and contains information specific to the bucket. It is described by an OB CRD, also shown below. An OBC references a storage class which is created by an administrator. The storage class defines whether the bucket requested is a new bucket or an existing bucket. It also defines the bucket retention policy. Users request a new or existing bucket by creating an OBC which is shown below. The ceph provisioner detects the OBC and creates a new bucket or grants access to an existing bucket, depending the the storage class referenced in the OBC. It also generates a Secret which provides credentials to access the bucket, and a ConfigMap which contains the bucket's endpoint. Application pods consume the information in the Secret and ConfigMap to access the bucket. Please note that to make provisioner watch the cluster namespace only you need to set `ROOK_OBC_WATCH_OPERATOR_NAMESPACE` to `true` in the operator manifest, otherwise it watches all namespaces. -## Sample +## Example ### OBC Custom Resource + ```yaml apiVersion: objectbucket.io/v1alpha1 kind: ObjectBucketClaim @@ -31,6 +28,7 @@ spec: maxObjects: "1000" maxSize: "2G" ``` + 1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap. 1. `namespace`(optional) of the `ObjectBucketClaim`, which is also the namespace of the ConfigMap and Secret. 1. `bucketName` name of the `bucket`. @@ -41,10 +39,12 @@ If both `bucketName` and `generateBucketName` are supplied then `BucketName` has If both `bucketName` and `generateBucketName` are blank or omitted then the storage class is expected to contain the name of an _existing_ bucket. It's an error if all three bucket related names are blank or omitted. 1. `storageClassName` which defines the StorageClass which contains the names of the bucket provisioner, the object-store and specifies the bucket retention policy. 1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Options supported: - - `maxObjects`: The maximum number of objects in the bucket - - `maxSize`: The maximum size of the bucket, please note minimum recommended value is 4K. + + * `maxObjects`: The maximum number of objects in the bucket + * `maxSize`: The maximum size of the bucket, please note minimum recommended value is 4K. ### OBC Custom Resource after Bucket Provisioning + ```yaml apiVersion: objectbucket.io/v1alpha1 kind: ObjectBucketClaim @@ -62,17 +62,19 @@ spec: status: phase: Bound [5] ``` + 1. `namespace` where OBC got created. 1. `ObjectBucketName` generated OB name created using name space and OBC name. 1. the generated (in this case), unique `bucket name` for the new bucket. 1. name of the storage class from OBC got created. 1. phases of bucket creation: - - _Pending_: the operator is processing the request. - - _Bound_: the operator finished processing the request and linked the OBC and OB - - _Released_: the OB has been deleted, leaving the OBC unclaimed but unavailable. - - _Failed_: not currently set. + * _Pending_: the operator is processing the request. + * _Bound_: the operator finished processing the request and linked the OBC and OB + * _Released_: the OB has been deleted, leaving the OBC unclaimed but unavailable. + * _Failed_: not currently set. ### App Pod + ```yaml apiVersion: v1 kind: Pod @@ -89,11 +91,13 @@ spec: - secretRef: name: ceph-bucket [3] ``` + 1. use `env:` if mapping of the defined key names to the env var names used by the app is needed. 1. makes available to the pod as env variables: `BUCKET_HOST`, `BUCKET_PORT`, `BUCKET_NAME` 1. makes available to the pod as env variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` ### StorageClass + ```yaml apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -108,11 +112,13 @@ parameters: [3] bucketName: ceph-bucket [4] reclaimPolicy: Delete [5] ``` + 1. `label`(optional) here associates this `StorageClass` to a specific provisioner. 1. `provisioner` responsible for handling `OBCs` referencing this `StorageClass`. 1. **all** `parameter` required. 1. `bucketName` is required for access to existing buckets but is omitted when provisioning new buckets. Unlike greenfield provisioning, the brownfield bucket name appears in the `StorageClass`, not the `OBC`. 1. rook-ceph provisioner decides how to treat the `reclaimPolicy` when an `OBC` is deleted for the bucket. See explanation as [specified in Kubernetes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain) -+ _Delete_ = physically delete the bucket. -+ _Retain_ = do not physically delete the bucket. + + * _Delete_ = physically delete the bucket. + * _Retain_ = do not physically delete the bucket. diff --git a/Documentation/ceph-object-bucket-notifications.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md similarity index 82% rename from Documentation/ceph-object-bucket-notifications.md rename to Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md index d8f5b18fed98..021f71ca598c 100644 --- a/Documentation/ceph-object-bucket-notifications.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md @@ -1,40 +1,43 @@ --- -title: Bucket Notifications -weight: 2950 -indent: true +title: Object Bucket Notifications --- -# Ceph Object Bucket Notifications - -> This feature is **experimental** +!!! info + This feature is **experimental** Rook supports the creation of bucket notifications via two custom resources: -- a `CephBucketNotification` is a custom resource the defines: topic, events and filters of a bucket notification, and is described by a Custom Resource Definition (CRD) shown below. Bucket notifications are associated with a bucket by setting labels on the Object Bucket claim (OBC). -See the Ceph documentation for detailed information: https://docs.ceph.com/en/latest/radosgw/notifications/. -- a `CephBucketTopic` is custom resource which represents a bucket notification topic and is described by a CRD shown below. A bucket notification topic represents an endpoint (or a "topic" inside this endpoint) to which bucket notifications could be sent. +* a `CephBucketNotification` is a custom resource the defines: topic, events and filters of a bucket notification, and is described by a Custom Resource Definition (CRD) shown below. Bucket notifications are associated with a bucket by setting labels on the Object Bucket claim (OBC). +See the Ceph documentation for detailed information: [Bucket Notifications - Ceph Object Gateway - Ceph Documentation](https://docs.ceph.com/en/latest/radosgw/notifications/). +* a `CephBucketTopic` is custom resource which represents a bucket notification topic and is described by a CRD shown below. A bucket notification topic represents an endpoint (or a "topic" inside this endpoint) to which bucket notifications could be sent. ## Notifications + A CephBucketNotification defines what bucket actions trigger the notification and which topic to send notifications to. A CephBucketNotification may also define a filter, based on the object's name and other object attributes. Notifications can be associated with buckets created via ObjectBucketClaims by adding labels to an ObjectBucketClaim with the following format: + ```yaml bucket-notification-: ``` + The CephBucketTopic, CephBucketNotification and ObjectBucketClaim must all belong to the same namespace. If a bucket was created manually (not via an ObjectBucketClaim), notifications on this bucket should also be created manually. However, topics in these notifications may reference topics that were created via CephBucketTopic resources. ## Topics + A CephBucketTopic represents an endpoint (of types: Kafka, AMQP0.9.1 or HTTP), or a specific resource inside this endpoint (e.g a Kafka or an AMQP topic, or a specific URI in an HTTP server). The CephBucketTopic also holds any additional info needed for a CephObjectStore's RADOS Gateways (RGW) to connect to the endpoint. Topics don't belong to a specific bucket or notification. Notifications from multiple buckets may be sent to the same topic, and one bucket (via multiple CephBucketNotifications) may send notifications to multiple topics. ## Notification Reliability and Delivery + Notifications may be sent synchronously, as part of the operation that triggered them. In this mode, the operation is acknowledged only after the notification is sent to the topic’s configured endpoint, which means that the round trip time of the notification is added to the latency of the operation itself. The original triggering operation will still be considered as successful even if the notification fail with an error, cannot be delivered or times out. Notifications may also be sent asynchronously. They will be committed into persistent storage and then asynchronously sent to the topic’s configured endpoint. In this case, the only latency added to the original operation is of committing the notification to persistent storage. If the notification fail with an error, cannot be delivered or times out, it will be retried until successfully acknowledged. -## Sample +## Example ### CephBucketTopic Custom Resource + ```yaml apiVersion: objectbucket.io/v1alpha1 kind: CephBucketTopic @@ -65,11 +68,11 @@ spec: # disableVerifySSL: true [16] # ackLevel: broker [17] # useSSL: false [18] - ``` + 1. `name` of the `CephBucketTopic` - - In case of AMQP endpoint, the name is used for the AMQP topic (“routing key” for a topic exchange) - - In case of Kafka endpoint, the name is used as the Kafka topic + * In case of AMQP endpoint, the name is used for the AMQP topic (“routing key” for a topic exchange) + * In case of Kafka endpoint, the name is used as the Kafka topic 1. `namespace`(optional) of the `CephBucketTopic`. Should match the namespace of the CephBucketNotification associated with this CephBucketTopic, and the OBC with the label referencing the CephBucketNotification 1. `objectStoreName` is the name of the object store in which the topic should be created. This must be the same object store used for the buckets associated with the notifications referencing this topic. 1. `objectStoreNamespace` is the namespace of the object store in which the topic should be created @@ -77,32 +80,35 @@ spec: 1. `persistent` (optional) indicates whether notifications to this endpoint are persistent (=asynchronous) or sent synchronously (“false” by default) 1. `endpoint` to which to send the notifications to. Exactly one of the endpoints must be defined: `http`, `amqp`, `kafka` 1. `http` (optional) hold the spec for an HTTP endpoint. The format of the URI would be: `http[s]://[:][/]` - - port defaults to: 80/443 for HTTP/S accordingly + * port defaults to: 80/443 for HTTP/S accordingly 1. `disableVerifySSL` indicates whether the RGW is going to verify the SSL certificate of the HTTP server in case HTTPS is used ("false" by default) 1. `amqp` (optional) hold the spec for an AMQP endpoint. The format of the URI would be: `amqp[s]://[:@][:][/]` - - port defaults to: 5672/5671 for AMQP/S accordingly - - user/password defaults to: guest/guest - - user/password may only be provided if HTTPS is used with the RGW. If not, topic creation request will be rejected - - vhost defaults to: “/” + * port defaults to: 5672/5671 for AMQP/S accordingly + * user/password defaults to: guest/guest + * user/password may only be provided if HTTPS is used with the RGW. If not, topic creation request will be rejected + * vhost defaults to: “/” 1. `disableVerifySSL` (optional) indicates whether the RGW is going to verify the SSL certificate of the AMQP server in case AMQPS is used ("false" by default) 1. `sendCloudEvents`: (optional) send the notifications with the [CloudEvents header](https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md). Supported for Ceph Quincy (v17) or newer ("false" by default) 1. `ackLevel` (optional) indicates what kind of ack the RGW is waiting for after sending the notifications: - - “none”: message is considered “delivered” if sent to broker - - “broker”: message is considered “delivered” if acked by broker (default) - - “routable”: message is considered “delivered” if broker can route to a consumer + * “none”: message is considered “delivered” if sent to broker + * “broker”: message is considered “delivered” if acked by broker (default) + * “routable”: message is considered “delivered” if broker can route to a consumer 1. `exchange` in the AMQP broker that would route the notifications. Different topics pointing to the same endpoint must use the same exchange 1. `kafka` (optional) hold the spec for a Kafka endpoint. The format of the URI would be: `kafka://[:@][: Note that n case of Kafka and AMQP, the consumer of the notifications is not required to ack the notifications, since the broker persists the messages before delivering them to their final destinations + +!!! note + In case of Kafka and AMQP, the consumer of the notifications is not required to ack the notifications, since the broker persists the messages before delivering them to their final destinations. ### CephBucketNotification Custom Resource + ```yaml apiVersion: ceph.rook.io/v1 kind: CephBucketNotification @@ -137,6 +143,7 @@ spec: - s3:ObjectCreated:Put - s3:ObjectCreated:Copy ``` + 1. `name` of the `CephBucketNotification` 1. `namespace`(optional) of the `CephBucketNotification`. Should match the namespace of the CephBucketTopic referenced in [3], and the OBC with the label referencing the CephBucketNotification 1. `topic` to which the notifications should be sent @@ -145,19 +152,21 @@ spec: 1. `metadataFilters` (optional) are filters based on the object metadata. All metadata fields defined as filters must exists in the object, with the values defined in the filter. Other metadata fields may exist in the object 1. `tagFilters` (optional) are filters based on object tags. All tags defined as filters must exists in the object, with the values defined in the filter. Other tags may exist in the object 1. `events` (optional) is a list of events that should trigger the notifications. By default all events should trigger notifications. Valid Events are: - - s3:ObjectCreated:* - - s3:ObjectCreated:Put - - s3:ObjectCreated:Post - - s3:ObjectCreated:Copy - - s3:ObjectCreated:CompleteMultipartUpload - - s3:ObjectRemoved:* - - s3:ObjectRemoved:Delete - - s3:ObjectRemoved:DeleteMarkerCreated + * s3:ObjectCreated:* + * s3:ObjectCreated:Put + * s3:ObjectCreated:Post + * s3:ObjectCreated:Copy + * s3:ObjectCreated:CompleteMultipartUpload + * s3:ObjectRemoved:* + * s3:ObjectRemoved:Delete + * s3:ObjectRemoved:DeleteMarkerCreated ### OBC Custom Resource + For a notifications to be associated with a bucket, a labels must be added to the OBC, indicating the name of the notification. To delete a notification from a bucket the matching label must be removed. When an OBC is deleted, all of the notifications associated with the bucket will be deleted as well. + ```yaml apiVersion: objectbucket.io/v1alpha1 kind: ObjectBucketClaim diff --git a/Documentation/ceph-object-multisite.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md similarity index 79% rename from Documentation/ceph-object-multisite.md rename to Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md index b6270ec23624..db3ebc9ef434 100644 --- a/Documentation/ceph-object-multisite.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md @@ -1,11 +1,6 @@ --- -title: Object Multisite -weight: 2250 -indent: true +title: Object Store Multisite --- -{% include_relative branch.liquid %} - -# Object Multisite Multisite is a feature of Ceph that allows object stores to replicate their data over multiple Ceph clusters. @@ -20,22 +15,22 @@ When a ceph-object-store is created with the `zone` section, the ceph-object-sto This allows the ceph-object-store to replicate its data over multiple Ceph clusters. To review core multisite concepts please read the -[ceph-multisite design overview](https://github.com/rook/rook/blob/{{ branchName }}/design/ceph/object/ceph-multisite-overview.md). +[ceph-multisite design overview](https://github.com/rook/rook/blob/master/design/ceph/object/ceph-multisite-overview.md). -## Prerequisites +### Prerequisites -This guide assumes a Rook cluster as explained in the [Quickstart](quickstart.md). +This guide assumes a Rook cluster as explained in the [Quickstart](../../Getting-Started/quickstart.md). -# Creating Object Multisite +## Creating Object Multisite If an admin wants to set up multisite on a Rook Ceph cluster, the admin should create: -1. A [realm](ceph-object-multisite-crd.md#object-realm-settings) -1. A [zonegroup](ceph-object-multisite-crd.md#object-zone-group-settings) -1. A [zone](ceph-object-multisite-crd.md#object-zone-settings) +1. A [realm](../../CRDs/Object-Storage/ceph-object-multisite-crd.md#object-realm-settings) +1. A [zonegroup](../../CRDs/Object-Storage/ceph-object-multisite-crd.md#object-zone-group-settings) +1. A [zone](../../CRDs/Object-Storage/ceph-object-multisite-crd.md#object-zone-settings) 1. A ceph object store with the `zone` section -object-multisite.yaml in the [examples](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/) directory can be used to create the multisite CRDs. +object-multisite.yaml in the [examples](https://github.com/rook/rook/blob/master/deploy/examples/) directory can be used to create the multisite CRDs. ```console kubectl create -f object-multisite.yaml @@ -49,9 +44,9 @@ The zone will create the pools for the object-store(s) that are in the zone to u When one of the multisite CRs (realm, zone group, zone) is deleted the underlying ceph realm/zone group/zone is not deleted, neither are the pools created by the zone. See the "Multisite Cleanup" section for more information. -For more information on the multisite CRDs please read [ceph-object-multisite-crd](ceph-object-multisite-crd.md). +For more information on the multisite CRDs please read [ceph-object-multisite-crd](../../CRDs/Object-Storage/ceph-object-multisite-crd.md). -# Pulling a Realm +## Pulling a Realm If an admin wants to sync data from another cluster, the admin needs to pull a realm on a Rook Ceph cluster from another Rook Ceph (or Ceph) cluster. @@ -60,33 +55,29 @@ To begin doing this, the admin needs 2 pieces of information: 1. An endpoint from the realm being pulled from 1. The access key and the system key of the system user from the realm being pulled from. -## Getting the Pull Endpoint +### Getting the Pull Endpoint To pull a Ceph realm from a remote Ceph cluster, an `endpoint` must be added to the CephObjectRealm's `pull` section in the `spec`. This endpoint must be from the master zone in the master zone group of that realm. If an admin does not know of an endpoint that fits this criteria, the admin can find such an endpoint on the remote Ceph cluster (via the tool box if it is a Rook Ceph Cluster) by running: -``` +```console $ radosgw-admin zonegroup get --rgw-realm=$REALM_NAME --rgw-zonegroup=$MASTER_ZONEGROUP_NAME +{ + ... + "endpoints": [http://10.17.159.77:80], + ... +} ``` ->``` ->{ -> ... -> "endpoints": [http://10.17.159.77:80], -> ... ->} ->``` A list of endpoints in the master zone group in the master zone is in the `endpoints` section of the JSON output of the `zonegoup get` command. This endpoint must also be resolvable from the new Rook Ceph cluster. To test this run the `curl` command on the endpoint: -``` +```console $ curl -L http://10.17.159.77:80 +anonymous ``` ->``` ->anonymous ->``` Finally add the endpoint to the `pull` section of the CephObjectRealm's spec. The CephObjectRealm should have the same name as the CephObjectRealm/Ceph realm it is pulling from. @@ -101,11 +92,11 @@ spec: endpoint: http://10.17.159.77:80 ``` -## Getting Realm Access Key and Secret Key +### Getting Realm Access Key and Secret Key The access key and secret key of the system user are keys that allow other Ceph clusters to pull the realm of the system user. -### Getting the Realm Access Key and Secret Key from the Rook Ceph Cluster +#### Getting the Realm Access Key and Secret Key from the Rook Ceph Cluster When an admin creates a ceph-object-realm a system user automatically gets created for the realm with an access key and a secret key. @@ -114,38 +105,45 @@ This system user has the name "$REALM_NAME-system-user". For the example realm, These keys for the user are exported as a kubernetes [secret](https://kubernetes.io/docs/concepts/configuration/secret/) called "$REALM_NAME-keys" (ex: realm-a-keys). To get these keys from the cluster the realm was originally created on, run: + ```console -$ kubectl -n $ORIGINAL_CLUSTER_NAMESPACE get secrets realm-a-keys -o yaml > realm-a-keys.yaml +kubectl -n $ORIGINAL_CLUSTER_NAMESPACE get secrets realm-a-keys -o yaml > realm-a-keys.yaml ``` + Edit the `realm-a-keys.yaml` file, and change the `namespace` with the namespace that the new Rook Ceph cluster exists in. Then create a kubernetes secret on the pulling Rook Ceph cluster with the same secrets yaml file. + ```console kubectl create -f realm-a-keys.yaml ``` -### Getting the Realm Access Key and Secret Key from a Non Rook Ceph Cluster +#### Getting the Realm Access Key and Secret Key from a Non Rook Ceph Cluster The access key and the secret key of the system user can be found in the output of running the following command on a non-rook ceph cluster: -``` + +```console radosgw-admin user info --uid="realm-a-system-user" ``` ->```{ -> ... -> "keys": [ -> { -> "user": "realm-a-system-user" -> "access_key": "aSw4blZIKV9nKEU5VC0=" -> "secret_key": "JSlDXFt5TlgjSV9QOE9XUndrLiI5JEo9YDBsJg==", -> } -> ], -> ... ->} ->``` + +```json +{ + ... + "keys": [ + { + "user": "realm-a-system-user" + "access_key": "aSw4blZIKV9nKEU5VC0=" + "secret_key": "JSlDXFt5TlgjSV9QOE9XUndrLiI5JEo9YDBsJg==", + } + ], + ... +} +``` Then base64 encode the each of the keys and create a `.yaml` file for the Kubernetes secret from the following template. Only the `access-key`, `secret-key`, and `namespace` sections need to be replaced. + ```yaml apiVersion: v1 data: @@ -159,36 +157,37 @@ type: kubernetes.io/rook ``` Finally, create a kubernetes secret on the pulling Rook Ceph cluster with the new secrets yaml file. + ```console kubectl create -f realm-a-keys.yaml ``` -### Pulling a Realm on a New Rook Ceph Cluster +#### Pulling a Realm on a New Rook Ceph Cluster Once the admin knows the endpoint and the secret for the keys has been created, the admin should create: -1. A [CephObjectRealm](ceph-object-multisite-crd.md#object-realm-settings) matching to the realm on the other Ceph cluster, with an endpoint as described above. -1. A [CephObjectZoneGroup](ceph-object-multisite-crd.md#object-zone-group-settings) matching the master zone group name or the master CephObjectZoneGroup from the cluster the the realm was pulled from. -1. A [CephObjectZone](ceph-object-multisite-crd.md#object-zone-settings) referring to the CephObjectZoneGroup created above. +1. A [CephObjectRealm](../../CRDs/Object-Storage/ceph-object-multisite-crd.md#object-realm-settings) matching to the realm on the other Ceph cluster, with an endpoint as described above. +1. A [CephObjectZoneGroup](../../CRDs/Object-Storage/ceph-object-multisite-crd.md#object-zone-group-settings) matching the master zone group name or the master CephObjectZoneGroup from the cluster the the realm was pulled from. +1. A [CephObjectZone](../../CRDs/Object-Storage/ceph-object-multisite-crd.md#object-zone-settings) referring to the CephObjectZoneGroup created above. 1. A CephObjectStore referring to the new CephObjectZone resource. -object-multisite-pull-realm.yaml (with changes) in the [examples](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/) directory can be used to create the multisite CRDs. +object-multisite-pull-realm.yaml (with changes) in the [examples](https://github.com/rook/rook/blob/master/deploy/examples/) directory can be used to create the multisite CRDs. ```console kubectl create -f object-multisite-pull-realm.yaml ``` -# Multisite Cleanup +## Multisite Cleanup Multisite configuration must be cleaned up by hand. Deleting a realm/zone group/zone CR will not delete the underlying Ceph realm, zone group, zone, or the pools associated with a zone. -## Realm Deletion +### Realm Deletion Changes made to the resource's configuration or deletion of the resource are not reflected on the Ceph cluster. When the ceph-object-realm resource is deleted or modified, the realm is not deleted from the Ceph cluster. Realm deletion must be done via the toolbox. -### Deleting a Realm +#### Deleting a Realm The Rook toolbox can modify the Ceph Multisite state via the radosgw-admin command. @@ -198,13 +197,13 @@ The following command, run via the toolbox, deletes the realm. radosgw-admin realm delete --rgw-realm=realm-a ``` -## Zone Group Deletion +### Zone Group Deletion Changes made to the resource's configuration or deletion of the resource are not reflected on the Ceph cluster. When the ceph-object-zone group resource is deleted or modified, the zone group is not deleted from the Ceph cluster. Zone Group deletion must be done through the toolbox. -### Deleting a Zone Group +#### Deleting a Zone Group The Rook toolbox can modify the Ceph Multisite state via the radosgw-admin command. @@ -215,13 +214,13 @@ radosgw-admin zonegroup delete --rgw-realm=realm-a --rgw-zonegroup=zone-group-a radosgw-admin period update --commit --rgw-realm=realm-a --rgw-zonegroup=zone-group-a ``` -## Deleting and Reconfiguring the Ceph Object Zone +### Deleting and Reconfiguring the Ceph Object Zone Changes made to the resource's configuration or deletion of the resource are not reflected on the Ceph cluster. When the ceph-object-zone resource is deleted or modified, the zone is not deleted from the Ceph cluster. Zone deletion must be done through the toolbox. -### Changing the Master Zone +#### Changing the Master Zone The Rook toolbox can change the master zone in a zone group. @@ -231,7 +230,7 @@ radosgw-admin zonegroup modify --rgw-realm=realm-a --rgw-zonegroup=zone-group-a radosgw-admin period update --commit --rgw-realm=realm-a --rgw-zonegroup=zone-group-a --rgw-zone=zone-a ``` -### Deleting Zone +#### Deleting Zone The Rook toolbox can modify the Ceph Multisite state via the radosgw-admin command. @@ -260,14 +259,15 @@ radosgw-admin period update --commit --rgw-realm=realm-a --rgw-zonegroup=zone-gr When a zone is deleted, the pools for that zone are not deleted. -### Deleting Pools for a Zone +#### Deleting Pools for a Zone The Rook toolbox can delete pools. Deleting pools should be done with caution. The following [documentation](https://docs.ceph.com/docs/master/rados/operations/pools/) on pools should be read before deleting any pools. When a zone is created the following pools are created for each zone: -``` + +```console $ZONE_NAME.rgw.control $ZONE_NAME.rgw.meta $ZONE_NAME.rgw.log @@ -275,6 +275,7 @@ $ZONE_NAME.rgw.buckets.index $ZONE_NAME.rgw.buckets.non-ec $ZONE_NAME.rgw.buckets.data ``` + Here is an example command to delete the .rgw.buckets.data pool for zone-a. ```console @@ -283,9 +284,10 @@ ceph osd pool rm zone-a.rgw.buckets.data zone-a.rgw.buckets.data --yes-i-really- In this command the pool name **must** be mentioned twice for the pool to be removed. -### Removing an Object Store from a Zone +#### Removing an Object Store from a Zone When an object-store (created in a zone) is deleted, the endpoint for that object store is removed from that zone, via + ```console kubectl delete -f object-store.yaml ``` diff --git a/Documentation/ceph-object.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md similarity index 82% rename from Documentation/ceph-object.md rename to Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md index 5d6b68f52cef..5cb77e2adbd3 100644 --- a/Documentation/ceph-object.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md @@ -1,16 +1,12 @@ --- -title: Object Storage -weight: 2200 -indent: true +title: Object Storage Overview --- -# Object Storage - Object storage exposes an S3 API to the storage cluster for applications to put and get data. ## Prerequisites -This guide assumes a Rook cluster as explained in the [Quickstart](quickstart.md). +This guide assumes a Rook cluster as explained in the [Quickstart](../../Getting-Started/quickstart.md). ## Configure an Object Store @@ -23,11 +19,12 @@ Alternatively, if you have an existing Ceph cluster with Rados Gateways, see the The below sample will create a `CephObjectStore` that starts the RGW service in the cluster with an S3 API. -> **NOTE**: This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. +!!! note + This sample requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. -The OSDs must be located on different nodes, because the [`failureDomain`](ceph-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). +The OSDs must be located on different nodes, because the [`failureDomain`](../../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). -See the [Object Store CRD](ceph-object-store-crd.md#object-store-settings), for more detail on the settings available for a `CephObjectStore`. +See the [Object Store CRD](../../CRDs/Object-Storage/ceph-object-store-crd.md#object-store-settings), for more detail on the settings available for a `CephObjectStore`. ```yaml apiVersion: ceph.rook.io/v1 @@ -59,11 +56,15 @@ spec: After the `CephObjectStore` is created, the Rook operator will then create all the pools and other resources necessary to start the service. This may take a minute to complete. +Create an object store: + ```console -# Create the object store kubectl create -f object.yaml +``` -# To confirm the object store is configured, wait for the rgw pod to start +To confirm the object store is configured, wait for the RGW pod(s) to start: + +```console kubectl -n rook-ceph get pod -l app=rook-ceph-rgw ``` @@ -92,38 +93,29 @@ spec: You can use the existing `object-external.yaml` file. When ready the ceph-object-controller will output a message in the Operator log similar to this one: ->``` ->ceph-object-controller: ceph object store gateway service >running at 10.100.28.138:8080 ->``` +```console +ceph-object-controller: ceph object store gateway service >running at 10.100.28.138:8080 +``` You can now get and access the store via: ```console -kubectl -n rook-ceph get svc -l app=rook-ceph-rgw +$ kubectl -n rook-ceph get svc -l app=rook-ceph-rgw +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-rgw-my-store ClusterIP 10.100.28.138 8080/TCP 6h59m ``` ->``` ->NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->rook-ceph-rgw-my-store ClusterIP 10.100.28.138 8080/TCP 6h59m ->``` - Any pod from your cluster can now access this endpoint: ```console $ curl 10.100.28.138:8080 +anonymous ``` ->``` ->anonymous ->``` - It is also possible to use the internally registered DNS name: ```console -curl rook-ceph-rgw-my-store.rook-ceph:8080 -``` - -```console +$ curl rook-ceph-rgw-my-store.rook-ceph:8080 anonymous ``` @@ -147,7 +139,9 @@ parameters: objectStoreName: my-store objectStoreNamespace: rook-ceph ``` + If you’ve deployed the Rook operator in a namespace other than `rook-ceph`, change the prefix in the provisioner to match the namespace you used. For example, if the Rook operator is running in the namespace `my-namespace` the provisioner value should be `my-namespace.ceph.rook.io/bucket`. + ```console kubectl create -f storageclass-bucket-delete.yaml ``` @@ -180,7 +174,7 @@ See the [Object Bucket Claim Documentation](ceph-object-bucket-claim.md) for mor The following commands extract key pieces of information from the secret and configmap:" -```bash +```console #config-map, secret, OBC will part of default if no specific name space mentioned export AWS_HOST=$(kubectl -n default get cm ceph-bucket -o jsonpath='{.data.BUCKET_HOST}') export PORT=$(kubectl -n default get cm ceph-bucket -o jsonpath='{.data.BUCKET_PORT}') @@ -195,14 +189,14 @@ Now that you have the object store configured and a bucket created, you can cons object storage from an S3 client. This section will guide you through testing the connection to the `CephObjectStore` and uploading and downloading from it. -Run the following commands after you have connected to the [Rook toolbox](ceph-toolbox.md). +Run the following commands after you have connected to the [Rook toolbox](../../Troubleshooting/ceph-toolbox.md). ### Connection Environment Variables To simplify the s3 client commands, you will want to set the four environment variables for use by your client (ie. inside the toolbox). See above for retrieving the variables for a bucket created by an `ObjectBucketClaim`. -```bash +```console export AWS_HOST= export PORT= export AWS_ACCESS_KEY_ID= @@ -216,7 +210,7 @@ export AWS_SECRET_ACCESS_KEY= The variables for the user generated in this example might be: -```bash +```console export AWS_HOST=rook-ceph-rgw-my-store.rook-ceph.svc export PORT=80 export AWS_ACCESS_KEY_ID=XEZDB3UJ6X7HVBE7X7MA @@ -263,14 +257,11 @@ you will need to setup an external service through a `NodePort`. First, note the service that exposes RGW internal to the cluster. We will leave this service intact and create a new service for external access. ```console -kubectl -n rook-ceph get service rook-ceph-rgw-my-store +$ kubectl -n rook-ceph get service rook-ceph-rgw-my-store +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-rgw-my-store 10.3.0.177 80/TCP 2m ``` ->``` ->NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->rook-ceph-rgw-my-store 10.3.0.177 80/TCP 2m ->``` - Save the external service as `rgw-external.yaml`: ```yaml @@ -306,15 +297,12 @@ kubectl create -f rgw-external.yaml See both rgw services running and notice what port the external service is running on: ```console -kubectl -n rook-ceph get service rook-ceph-rgw-my-store rook-ceph-rgw-my-store-external +$ kubectl -n rook-ceph get service rook-ceph-rgw-my-store rook-ceph-rgw-my-store-external +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-rgw-my-store ClusterIP 10.104.82.228 80/TCP 4m +rook-ceph-rgw-my-store-external NodePort 10.111.113.237 80:31536/TCP 39s ``` ->``` ->NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->rook-ceph-rgw-my-store ClusterIP 10.104.82.228 80/TCP 4m ->rook-ceph-rgw-my-store-external NodePort 10.111.113.237 80:31536/TCP 39s ->``` - Internally the rgw service is running on port `80`. The external port in this case is `31536`. Now you can access the `CephObjectStore` from anywhere! All you need is the hostname for any machine in the cluster, the external port, and the user credentials. ## Create a User @@ -324,7 +312,7 @@ create a `CephObjectStoreUser`. The user will be used to connect to the RGW serv The user will be independent of any object bucket claims that you might have created in the earlier instructions in this document. -See the [Object Store User CRD](ceph-object-store-user-crd.md) for more detail on the settings available for a `CephObjectStoreUser`. +See the [Object Store User CRD](../../CRDs/Object-Storage/ceph-object-store-user-crd.md) for more detail on the settings available for a `CephObjectStoreUser`. ```yaml apiVersion: ceph.rook.io/v1 @@ -346,25 +334,22 @@ kubectl create -f object-user.yaml ```console # To confirm the object store user is configured, describe the secret -kubectl -n rook-ceph describe secret rook-ceph-object-user-my-store-my-user +$ kubectl -n rook-ceph describe secret rook-ceph-object-user-my-store-my-user +Name: rook-ceph-object-user-my-store-my-user +Namespace: rook-ceph +Labels: app=rook-ceph-rgw + rook_cluster=rook-ceph + rook_object_store=my-store +Annotations: + +Type: kubernetes.io/rook + +Data +==== +AccessKey: 20 bytes +SecretKey: 40 bytes ``` ->``` ->Name: rook-ceph-object-user-my-store-my-user ->Namespace: rook-ceph ->Labels: app=rook-ceph-rgw -> rook_cluster=rook-ceph -> rook_object_store=my-store ->Annotations: -> ->Type: kubernetes.io/rook -> ->Data ->==== ->AccessKey: 20 bytes ->SecretKey: 40 bytes ->``` - The AccessKey and SecretKey data fields can be mounted in a pod as an environment variable. More information on consuming kubernetes secrets can be found in the [K8s secret documentation](https://kubernetes.io/docs/concepts/configuration/secret/) diff --git a/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/.pages b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/.pages new file mode 100644 index 000000000000..d87ef02cc390 --- /dev/null +++ b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/.pages @@ -0,0 +1,4 @@ +title: Shared Filesystem (CephFS) +nav: + - filesystem-storage.md + - ... diff --git a/Documentation/ceph-filesystem.md b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md similarity index 82% rename from Documentation/ceph-filesystem.md rename to Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md index d345eec2daef..972121ed11e5 100644 --- a/Documentation/ceph-filesystem.md +++ b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md @@ -1,19 +1,14 @@ --- -title: Shared Filesystem -weight: 2300 -indent: true +title: Filesystem Storage Overview --- -{% include_relative branch.liquid %} -# Shared Filesystem - -A shared filesystem can be mounted with read/write permission from multiple pods. This may be useful for applications which can be clustered using a shared filesystem. +A filesystem storage (also named shared filesystem) can be mounted with read/write permission from multiple pods. This may be useful for applications which can be clustered using a shared filesystem. This example runs a shared filesystem for the [kube-registry](https://github.com/kubernetes/kubernetes/tree/release-1.9/cluster/addons/registry). ## Prerequisites -This guide assumes you have created a Rook cluster as explained in the main [Kubernetes guide](quickstart.md) +This guide assumes you have created a Rook cluster as explained in the main [quickstart guide](../../Getting-Started/quickstart.md) ### Multiple Filesystems Support @@ -21,7 +16,7 @@ Multiple filesystems are supported as of the Ceph Pacific release. ## Create the Filesystem -Create the filesystem by specifying the desired settings for the metadata pool, data pools, and metadata server in the `CephFilesystem` CRD. In this example we create the metadata pool with replication of three and a single data pool with replication of three. For more options, see the documentation on [creating shared filesystems](ceph-filesystem-crd.md). +Create the filesystem by specifying the desired settings for the metadata pool, data pools, and metadata server in the `CephFilesystem` CRD. In this example we create the metadata pool with replication of three and a single data pool with replication of three. For more options, see the documentation on [creating shared filesystems](../../CRDs/Shared-Filesystem/ceph-filesystem-crd.md). Save this shared filesystem definition as `filesystem.yaml`: @@ -53,27 +48,23 @@ kubectl create -f filesystem.yaml [...] ``` +To confirm the filesystem is configured, wait for the mds pods to start: + ```console -# To confirm the filesystem is configured, wait for the mds pods to start -kubectl -n rook-ceph get pod -l app=rook-ceph-mds +$ kubectl -n rook-ceph get pod -l app=rook-ceph-mds +NAME READY STATUS RESTARTS AGE +rook-ceph-mds-myfs-7d59fdfcf4-h8kw9 1/1 Running 0 12s +rook-ceph-mds-myfs-7d59fdfcf4-kgkjp 1/1 Running 0 12s ``` ->``` ->NAME READY STATUS RESTARTS AGE ->rook-ceph-mds-myfs-7d59fdfcf4-h8kw9 1/1 Running 0 12s ->rook-ceph-mds-myfs-7d59fdfcf4-kgkjp 1/1 Running 0 12s ->``` - -To see detailed status of the filesystem, start and connect to the [Rook toolbox](ceph-toolbox.md). A new line will be shown with `ceph status` for the `mds` service. In this example, there is one active instance of MDS which is up, with one MDS instance in `standby-replay` mode in case of failover. +To see detailed status of the filesystem, start and connect to the [Rook toolbox](../../Troubleshooting/ceph-toolbox.md). A new line will be shown with `ceph status` for the `mds` service. In this example, there is one active instance of MDS which is up, with one MDS instance in `standby-replay` mode in case of failover. ```console -ceph status +$ ceph status +[...] + services: + mds: myfs-1/1/1 up {[myfs:0]=mzw58b=up:active}, 1 up:standby-replay ``` ->``` -> ... -> services: -> mds: myfs-1/1/1 up {[myfs:0]=mzw58b=up:active}, 1 up:standby-replay ->``` ## Provision Storage @@ -126,13 +117,15 @@ kubectl create -f deploy/examples/csi/cephfs/storageclass.yaml ## Quotas -> **IMPORTANT**: The CephFS CSI driver uses quotas to enforce the PVC size requested. +!!! attention + The CephFS CSI driver uses quotas to enforce the PVC size requested. + Only newer kernels support CephFS quotas (kernel version of at least 4.17). If you require quotas to be enforced and the kernel driver does not support it, you can disable the kernel driver and use the FUSE client. This can be done by setting `CSI_FORCE_CEPHFS_KERNEL_CLIENT: false` in the operator deployment (`operator.yaml`). However, it is important to know that when the FUSE client is enabled, there is an issue that during upgrade the application pods will be -disconnected from the mount and will need to be restarted. See the [upgrade guide](ceph-upgrade.md) +disconnected from the mount and will need to be restarted. See the [upgrade guide](../../Upgrade/rook-upgrade.md) for more details. ## Consume the Shared Filesystem: K8s Registry Sample @@ -225,7 +218,7 @@ If the Rook cluster has more than one filesystem and the application pod is sche ## Consume the Shared Filesystem: Toolbox -Once you have pushed an image to the registry (see the [instructions](https://github.com/kubernetes/kubernetes/tree/release-1.9/cluster/addons/registry) to expose and use the kube-registry), verify that kube-registry is using the filesystem that was configured above by mounting the shared filesystem in the toolbox pod. See the [Direct Filesystem](direct-tools.md#shared-filesystem-tools) topic for more details. +Once you have pushed an image to the registry (see the [instructions](https://github.com/kubernetes/kubernetes/tree/release-1.9/cluster/addons/registry) to expose and use the kube-registry), verify that kube-registry is using the filesystem that was configured above by mounting the shared filesystem in the toolbox pod. See the [Direct Filesystem](../../Troubleshooting/direct-tools.md#shared-filesystem-tools) topic for more details. ## Teardown @@ -237,7 +230,8 @@ kubectl delete -f kube-registry.yaml To delete the filesystem components and backing data, delete the Filesystem CRD. -> **WARNING: Data will be deleted if preserveFilesystemOnDelete=false**. +!!! warning + Data will be deleted if preserveFilesystemOnDelete=false**. ```console kubectl -n rook-ceph delete cephfilesystem myfs @@ -247,4 +241,4 @@ Note: If the "preserveFilesystemOnDelete" filesystem attribute is set to true, t ### Advanced Example: Erasure Coded Filesystem -The Ceph filesystem example can be found here: [Ceph Shared Filesystem - Samples - Erasure Coded](ceph-filesystem-crd.md#erasure-coded). +The Ceph filesystem example can be found here: [Ceph Shared Filesystem - Samples - Erasure Coded](../../CRDs/Shared-Filesystem/ceph-filesystem-crd.md#erasure-coded). diff --git a/Documentation/ceph-teardown.md b/Documentation/Storage-Configuration/ceph-teardown.md similarity index 84% rename from Documentation/ceph-teardown.md rename to Documentation/Storage-Configuration/ceph-teardown.md index c3d2f0232ade..01eb5628d561 100644 --- a/Documentation/ceph-teardown.md +++ b/Documentation/Storage-Configuration/ceph-teardown.md @@ -1,10 +1,8 @@ --- title: Cleanup -weight: 3900 -indent: true --- -# Cleaning up a Cluster +## Cleaning up a Cluster If you want to tear down the cluster and bring up a new one, be aware of the following resources that will need to be cleaned up: @@ -17,11 +15,11 @@ If you see issues tearing down the cluster, see the [Troubleshooting](#troublesh If you are tearing down a cluster frequently for development purposes, it is instead recommended to use an environment such as Minikube that can easily be reset without worrying about any of these steps. -## Delete the Block and File artifacts +### Delete the Block and File artifacts First you will need to clean up the resources created on top of the Rook cluster. -These commands will clean up the resources from the [block](ceph-block.md#teardown) and [file](ceph-filesystem.md#teardown) walkthroughs (unmount volumes, delete volume claims, etc). If you did not complete those parts of the walkthrough, you can skip these instructions: +These commands will clean up the resources from the [block](Block-Storage-RBD/block-storage.md#teardown) and [file](Shared-Filesystem-CephFS/filesystem-storage.md#teardown) walkthroughs (unmount volumes, delete volume claims, etc). If you did not complete those parts of the walkthrough, you can skip these instructions: ```console kubectl delete -f ../wordpress.yaml @@ -34,7 +32,7 @@ kubectl delete storageclass csi-cephfs After those block and file resources have been cleaned up, you can then delete your Rook cluster. This is important to delete **before removing the Rook operator and agent or else resources may not be cleaned up properly**. -## Delete the CephCluster CRD +### Delete the CephCluster CRD Edit the `CephCluster` and add the `cleanupPolicy` @@ -46,7 +44,7 @@ kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"clea Once the cleanup policy is enabled, any new configuration changes in the CephCluster will be blocked. Nothing will happen until the deletion of the CR is requested, so this `cleanupPolicy` change can still be reverted if needed. -Checkout more details about the `cleanupPolicy` [here](ceph-cluster-crd.md#cleanup-policy) +Checkout more details about the `cleanupPolicy` [here](../CRDs/ceph-cluster-crd.md#cleanup-policy) Delete the `CephCluster` CR. @@ -62,12 +60,13 @@ kubectl -n rook-ceph get cephcluster If the `cleanupPolicy` was applied, then wait for the `rook-ceph-cleanup` jobs to be completed on all the nodes. These jobs will perform the following operations: -- Delete the directory `/var/lib/rook` (or the path specified by the `dataDirHostPath`) on all the nodes -- Wipe the data on the drives on all the nodes where OSDs were running in this cluster -Note: The cleanup jobs might not start if the resources created on top of Rook Cluster are not deleted completely. [See](ceph-teardown.md#delete-the-block-and-file-artifacts) +* Delete the directory `/var/lib/rook` (or the path specified by the `dataDirHostPath`) on all the nodes +* Wipe the data on the drives on all the nodes where OSDs were running in this cluster -## Delete the Operator and related Resources +Note: The cleanup jobs might not start if the resources created on top of Rook Cluster are not deleted completely. [See](#delete-the-block-and-file-artifacts) + +### Delete the Operator and related Resources This will begin the process of the Rook Ceph operator and all other resources being cleaned up. This includes related resources such as the agent and discover daemonsets with the following commands: @@ -80,9 +79,10 @@ kubectl delete -f crds.yaml If the `cleanupPolicy` was applied and the cleanup jobs have completed on all the nodes, then the cluster tear down has been successful. If you skipped adding the `cleanupPolicy` then follow the manual steps mentioned below to tear down the cluster. -## Delete the data on hosts +### Delete the data on hosts -> **IMPORTANT**: The final cleanup step requires deleting files on each host in the cluster. All files under the `dataDirHostPath` property specified in the cluster CRD will need to be deleted. Otherwise, inconsistent state will remain when a new cluster is started. +!!! attention + The final cleanup step requires deleting files on each host in the cluster. All files under the `dataDirHostPath` property specified in the cluster CRD will need to be deleted. Otherwise, inconsistent state will remain when a new cluster is started. Connect to each machine and delete `/var/lib/rook`, or the path specified by the `dataDirHostPath`. @@ -90,7 +90,7 @@ In the future this step will not be necessary when we build on the K8s local sto If you modified the demo settings, additional cleanup is up to you for devices, host paths, etc. -### Zapping Devices +#### Zapping Devices Disks on nodes used by Rook for osds can be reset to a usable state with methods suggested below. Note that these scripts are not one-size-fits-all. Please use them with discretion to ensure you are @@ -99,7 +99,7 @@ not removing data unrelated to Rook and/or Ceph. Disks can be zapped fairly easily. A single disk can usually be cleared with some or all of the steps below. -```sh +```console DISK="/dev/sdX" # Zap the disk to a fresh, usable state (zap-all is important, b/c MBR has to be clean) @@ -120,7 +120,7 @@ used again. These steps can help to free up old Ceph disks for re-use. Note that be run once on each node and assumes that **all** Ceph disks are being wiped. If only some disks are being wiped, you will have to manually determine which disks map to which device mapper devices. -```sh +```console # This command hangs on some systems: with caution, 'dmsetup remove_all --force' can be used ls /dev/mapper/ceph-* | xargs -I% -- dmsetup remove % @@ -131,7 +131,7 @@ rm -rf /dev/mapper/ceph--* If disks are still reported locked, rebooting the node often helps clear LVM-related holds on disks. -## Troubleshooting +### Troubleshooting If the cleanup instructions are not executed in the order above, or you otherwise have difficulty cleaning up the cluster, here are a few things to try. @@ -153,7 +153,7 @@ kubectl -n rook-ceph get cephcluster If the cluster CRD still exists even though you have executed the delete command earlier, see the next section on removing the finalizer. -### Removing the Cluster CRD Finalizer +#### Removing the Cluster CRD Finalizer When a Cluster CRD is created, a [finalizer](https://kubernetes.io/docs/tasks/access-kubernetes-api/extend-api-custom-resource-definitions/#finalizers) is added automatically by the Rook operator. The finalizer will allow the operator to ensure that before the cluster CRD is deleted, all block and file mounts will be cleaned up. Without proper cleanup, pods consuming the storage will be hung indefinitely until a system reboot. @@ -168,14 +168,15 @@ done ``` This command will patch the following CRDs on v1.3: ->``` -> cephblockpools.ceph.rook.io -> cephclients.ceph.rook.io -> cephfilesystems.ceph.rook.io -> cephnfses.ceph.rook.io -> cephobjectstores.ceph.rook.io -> cephobjectstoreusers.ceph.rook.io ->``` + +```console +cephblockpools.ceph.rook.io +cephclients.ceph.rook.io +cephfilesystems.ceph.rook.io +cephnfses.ceph.rook.io +cephobjectstores.ceph.rook.io +cephobjectstoreusers.ceph.rook.io +``` Within a few seconds you should see that the cluster CRD has been deleted and will no longer block other cleanup such as deleting the `rook-ceph` namespace. @@ -186,7 +187,7 @@ kubectl api-resources --verbs=list --namespaced -o name \ | xargs -n 1 kubectl get --show-kind --ignore-not-found -n rook-ceph ``` -### Remove critical resource finalizers +#### Remove critical resource finalizers Rook adds a finalizer `ceph.rook.io/disaster-protection` to resources critical to the Ceph cluster so that the resources will not be accidentally deleted. diff --git a/Documentation/Troubleshooting/.pages b/Documentation/Troubleshooting/.pages new file mode 100644 index 000000000000..971e3e85362c --- /dev/null +++ b/Documentation/Troubleshooting/.pages @@ -0,0 +1,9 @@ +nav: + - ceph-toolbox.md + - common-issues.md + - ceph-common-issues.md + - ceph-csi-common-issues.md + - openshift-common-issues.md + - disaster-recovery.md + - direct-tools.md + - ... diff --git a/Documentation/ceph-common-issues.md b/Documentation/Troubleshooting/ceph-common-issues.md similarity index 71% rename from Documentation/ceph-common-issues.md rename to Documentation/Troubleshooting/ceph-common-issues.md index 07c4bffd231b..3d287008ddad 100644 --- a/Documentation/ceph-common-issues.md +++ b/Documentation/Troubleshooting/ceph-common-issues.md @@ -1,35 +1,12 @@ --- -title: Common Issues -weight: 11120 -indent: true +title: Ceph Common Issues --- -# Ceph Common Issues - Many of these problem cases are hard to summarize down to a short phrase that adequately describes the problem. Each problem will start with a bulleted list of symptoms. Keep in mind that all symptoms may not apply depending on the configuration of Rook. If the majority of the symptoms are seen there is a fair chance you are experiencing that problem. If after trying the suggestions found on this page and the problem is not resolved, the Rook team is very happy to help you troubleshoot the issues in their Slack channel. Once you have [registered for the Rook Slack](https://slack.rook.io), proceed to the `#ceph` channel to ask for assistance. -## Table of Contents - -* [Troubleshooting Techniques](#troubleshooting-techniques) -* [Cluster failing to service requests](#cluster-failing-to-service-requests) -* [Monitors are the only pods running](#monitors-are-the-only-pods-running) -* [PVCs stay in pending state](#pvcs-stay-in-pending-state) -* [OSD pods are failing to start](#osd-pods-are-failing-to-start) -* [OSD pods are not created on my devices](#osd-pods-are-not-created-on-my-devices) -* [Node hangs after reboot](#node-hangs-after-reboot) -* [Using multiple shared filesystem (CephFS) is attempted on a kernel version older than 4.7](#using-multiple-shared-filesystem-cephfs-is-attempted-on-a-kernel-version-older-than-47) -* [Set debug log level for all Ceph daemons](#set-debug-log-level-for-all-ceph-daemons) -* [Activate log to file for a particular Ceph daemon](#activate-log-to-file-for-a-particular-ceph-daemon) -* [A worker node using RBD devices hangs up](#a-worker-node-using-rbd-devices-hangs-up) -* [Too few PGs per OSD warning is shown](#too-few-pgs-per-osd-warning-is-shown) -* [LVM metadata can be corrupted with OSD on LV-backed PVC](#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc) -* [OSD prepare job fails due to low aio-max-nr setting](#osd-prepare-job-fails-due-to-low-aio-max-nr-setting) -* [Unexpected partitions created](#unexpected-partitions-created) -* [Operator environment variables are ignored](#operator-environment-variables-are-ignored) - -See also the [CSI Troubleshooting Guide](ceph-csi-troubleshooting.md). +See also the [CSI Troubleshooting Guide](../Troubleshooting/ceph-csi-common-issues.md). ## Troubleshooting Techniques @@ -43,13 +20,13 @@ There are two main categories of information you will need to investigate issues After you verify the basic health of the running pods, next you will want to run Ceph tools for status of the storage components. There are two ways to run the Ceph tools, either in the Rook toolbox or inside other Rook pods that are already running. * Logs on a specific node to find why a PVC is failing to mount -* See the [log collection topic](ceph-advanced-configuration.md#log-collection) for a script that will help you gather the logs +* See the [log collection topic](../Storage-Configuration/Advanced/ceph-configuration.md#log-collection) for a script that will help you gather the logs * Other artifacts: * The monitors that are expected to be in quorum: `kubectl -n get configmap rook-ceph-mon-endpoints -o yaml | grep data` #### Tools in the Rook Toolbox -The [rook-ceph-tools pod](./ceph-toolbox.md) provides a simple environment to run Ceph tools. Once the pod is up and running, connect to the pod to execute Ceph commands to evaluate that current state of the cluster. +The [rook-ceph-tools pod](ceph-toolbox.md) provides a simple environment to run Ceph tools. Once the pod is up and running, connect to the pod to execute Ceph commands to evaluate that current state of the cluster. ```console kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[*].metadata.name}') bash @@ -69,7 +46,7 @@ Here are some common commands to troubleshoot a Ceph cluster: The first two status commands provide the overall cluster health. The normal state for cluster operations is HEALTH_OK, but will still function when the state is in a HEALTH_WARN state. If you are in a WARN state, then the cluster is in a condition that it may enter the HEALTH_ERROR state at which point *all* disk I/O operations are halted. If a HEALTH_WARN state is observed, then one should take action to prevent the cluster from halting when it enters the HEALTH_ERROR state. -There are many Ceph sub-commands to look at and manipulate Ceph objects, well beyond the scope this document. See the [Ceph documentation](https://docs.ceph.com/) for more details of gathering information about the health of the cluster. In addition, there are other helpful hints and some best practices located in the [Advanced Configuration section](ceph-advanced-configuration.md). Of particular note, there are scripts for collecting logs and gathering OSD information there. +There are many Ceph sub-commands to look at and manipulate Ceph objects, well beyond the scope this document. See the [Ceph documentation](https://docs.ceph.com/) for more details of gathering information about the health of the cluster. In addition, there are other helpful hints and some best practices located in the [Advanced Configuration section](../Storage-Configuration/Advanced/ceph-configuration.md). Of particular note, there are scripts for collecting logs and gathering OSD information there. ## Cluster failing to service requests @@ -95,24 +72,21 @@ ceph status Another indication is when one or more of the MON pods restart frequently. Note the 'mon107' that has only been up for 16 minutes in the following output. ```console -kubectl -n rook-ceph get all -o wide --show-all +$ kubectl -n rook-ceph get all -o wide --show-all +NAME READY STATUS RESTARTS AGE IP NODE +po/rook-ceph-mgr0-2487684371-gzlbq 1/1 Running 0 17h 192.168.224.46 k8-host-0402 +po/rook-ceph-mon107-p74rj 1/1 Running 0 16m 192.168.224.28 k8-host-0402 +rook-ceph-mon1-56fgm 1/1 Running 0 2d 192.168.91.135 k8-host-0404 +rook-ceph-mon2-rlxcd 1/1 Running 0 2d 192.168.123.33 k8-host-0403 +rook-ceph-osd-bg2vj 1/1 Running 0 2d 192.168.91.177 k8-host-0404 +rook-ceph-osd-mwxdm 1/1 Running 0 2d 192.168.123.31 k8-host-0403 ``` ->``` ->NAME READY STATUS RESTARTS AGE IP NODE ->po/rook-ceph-mgr0-2487684371-gzlbq 1/1 Running 0 17h 192.168.224.46 k8-host-0402 ->po/rook-ceph-mon107-p74rj 1/1 Running 0 16m 192.168.224.28 k8-host-0402 ->rook-ceph-mon1-56fgm 1/1 Running 0 2d 192.168.91.135 k8-host-0404 ->rook-ceph-mon2-rlxcd 1/1 Running 0 2d 192.168.123.33 k8-host-0403 ->rook-ceph-osd-bg2vj 1/1 Running 0 2d 192.168.91.177 k8-host-0404 ->rook-ceph-osd-mwxdm 1/1 Running 0 2d 192.168.123.31 k8-host-0403 ->``` - ### Solution What is happening here is that the MON pods are restarting and one or more of the Ceph daemons are not getting configured with the proper cluster information. This is commonly the result of not specifying a value for `dataDirHostPath` in your Cluster CRD. -The `dataDirHostPath` setting specifies a path on the local host for the Ceph daemons to store configuration and data. Setting this to a path like `/var/lib/rook`, reapplying your Cluster CRD and restarting all the Ceph daemons (MON, MGR, OSD, RGW) should solve this problem. After the Ceph daemons have been restarted, it is advisable to restart the [rook-tool pod](ceph-toolbox.md). +The `dataDirHostPath` setting specifies a path on the local host for the Ceph daemons to store configuration and data. Setting this to a path like `/var/lib/rook`, reapplying your Cluster CRD and restarting all the Ceph daemons (MON, MGR, OSD, RGW) should solve this problem. After the Ceph daemons have been restarted, it is advisable to restart the [rook-tools pod](ceph-toolbox.md). ## Monitors are the only pods running @@ -135,7 +109,7 @@ There are several common causes for the mons failing to form quorum: * The operator pod does not have network connectivity to the mon pod(s). The network may be configured incorrectly. * One or more mon pods are in running state, but the operator log shows they are not able to form quorum -* A mon is using configuration from a previous installation. See the [cleanup guide](ceph-teardown.md#delete-the-data-on-hosts) +* A mon is using configuration from a previous installation. See the [cleanup guide](../Storage-Configuration/ceph-teardown.md#delete-the-data-on-hosts) for cleaning the previous cluster. * A firewall may be blocking the ports required for the Ceph mons to form quorum. Ensure ports 6789 and 3300 are enabled. See the [Ceph networking guide](https://docs.ceph.com/en/latest/rados/configuration/network-config-ref/) for more details. @@ -155,14 +129,14 @@ kubectl -n rook-ceph logs -l app=rook-ceph-operator Likely you will see an error similar to the following that the operator is timing out when connecting to the mon. The last command is `ceph mon_status`, followed by a timeout message five minutes later. ->``` ->2018-01-21 21:47:32.375833 I | exec: Running command: ceph mon_status --cluster=rook --conf=/var/lib/rook/rook-ceph/rook.config --keyring=/var/lib/rook/rook-ceph/client.admin.keyring --format json --out-file /tmp/442263890 ->2018-01-21 21:52:35.370533 I | exec: 2018-01-21 21:52:35.071462 7f96a3b82700 0 monclient(hunting): authenticate timed out after 300 ->2018-01-21 21:52:35.071462 7f96a3b82700 0 monclient(hunting): authenticate timed out after 300 ->2018-01-21 21:52:35.071524 7f96a3b82700 0 librados: client.admin authentication error (110) Connection timed out ->2018-01-21 21:52:35.071524 7f96a3b82700 0 librados: client.admin authentication error (110) Connection timed out ->[errno 110] error connecting to the cluster ->``` +```console +2018-01-21 21:47:32.375833 I | exec: Running command: ceph mon_status --cluster=rook --conf=/var/lib/rook/rook-ceph/rook.config --keyring=/var/lib/rook/rook-ceph/client.admin.keyring --format json --out-file /tmp/442263890 +2018-01-21 21:52:35.370533 I | exec: 2018-01-21 21:52:35.071462 7f96a3b82700 0 monclient(hunting): authenticate timed out after 300 +2018-01-21 21:52:35.071462 7f96a3b82700 0 monclient(hunting): authenticate timed out after 300 +2018-01-21 21:52:35.071524 7f96a3b82700 0 librados: client.admin authentication error (110) Connection timed out +2018-01-21 21:52:35.071524 7f96a3b82700 0 librados: client.admin authentication error (110) Connection timed out +[errno 110] error connecting to the cluster +``` The error would appear to be an authentication error, but it is misleading. The real issue is a timeout. @@ -173,18 +147,16 @@ If the mon pod is running, check the network connectivity between the operator p A common issue is that the CNI is not configured correctly. To verify the network connectivity: -- Get the endpoint for a mon -- Curl the mon from the operator pod + +* Get the endpoint for a mon +* Curl the mon from the operator pod For example, this command will curl the first mon from the operator: +```console +$ kubectl -n rook-ceph exec deploy/rook-ceph-operator -- curl $(kubectl -n rook-ceph get svc -l app=rook-ceph-mon -o jsonpath='{.items[0].spec.clusterIP}'):3300 2>/dev/null +ceph v2 ``` -kubectl -n rook-ceph exec deploy/rook-ceph-operator -- curl $(kubectl -n rook-ceph get svc -l app=rook-ceph-mon -o jsonpath='{.items[0].spec.clusterIP}'):3300 2>/dev/null -``` - ->``` ->ceph v2 ->``` If "ceph v2" is printed to the console, the connection was successful. If the command does not respond or otherwise fails, the network connection cannot be established. @@ -194,31 +166,25 @@ otherwise fails, the network connection cannot be established. Second we need to verify if the mon pod started successfully. ```console -kubectl -n rook-ceph get pod -l app=rook-ceph-mon +$ kubectl -n rook-ceph get pod -l app=rook-ceph-mon +NAME READY STATUS RESTARTS AGE +rook-ceph-mon-a-69fb9c78cd-58szd 1/1 CrashLoopBackOff 2 47s ``` ->``` ->NAME READY STATUS RESTARTS AGE ->rook-ceph-mon-a-69fb9c78cd-58szd 1/1 CrashLoopBackOff 2 47s ->``` - If the mon pod is failing as in this example, you will need to look at the mon pod status or logs to determine the cause. If the pod is in a crash loop backoff state, you should see the reason by describing the pod. ```console # The pod shows a termination status that the keyring does not match the existing keyring -kubectl -n rook-ceph describe pod -l mon=rook-ceph-mon0 +$ kubectl -n rook-ceph describe pod -l mon=rook-ceph-mon0 +... + Last State: Terminated + Reason: Error + Message: The keyring does not match the existing keyring in /var/lib/rook/rook-ceph-mon0/data/keyring. + You may need to delete the contents of dataDirHostPath on the host from a previous deployment. +... ``` ->``` ->... -> Last State: Terminated -> Reason: Error -> Message: The keyring does not match the existing keyring in /var/lib/rook/rook-ceph-mon0/data/keyring. -> You may need to delete the contents of dataDirHostPath on the host from a previous deployment. ->... ->``` - See the solution in the next section regarding cleaning up the `dataDirHostPath` on the nodes. #### Solution @@ -228,9 +194,10 @@ This directory is the `dataDirHostPath` setting in the cluster CRD and is typica To fix the issue you will need to delete all components of Rook and then delete the contents of `/var/lib/rook` (or the directory specified by `dataDirHostPath`) on each of the hosts in the cluster. Then when the cluster CRD is applied to start a new cluster, the rook-operator should start all the pods as expected. -> **IMPORTANT: Deleting the `dataDirHostPath` folder is destructive to the storage. Only delete the folder if you are trying to permanently purge the Rook cluster.** +!!! caution + **Deleting the `dataDirHostPath` folder is destructive to the storage. Only delete the folder if you are trying to permanently purge the Rook cluster.** -See the [Cleanup Guide](ceph-teardown.md) for more details. +See the [Cleanup Guide](../Storage-Configuration/ceph-teardown.md) for more details. ## PVCs stay in pending state @@ -241,15 +208,12 @@ See the [Cleanup Guide](ceph-teardown.md) for more details. For the Wordpress example, you might see two PVCs in pending state. ```console -kubectl get pvc +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +mysql-pv-claim Pending rook-ceph-block 8s +wp-pv-claim Pending rook-ceph-block 16s ``` ->``` ->NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ->mysql-pv-claim Pending rook-ceph-block 8s ->wp-pv-claim Pending rook-ceph-block 16s ->``` - ### Investigation There are two common causes for the PVCs staying in pending state: @@ -265,19 +229,16 @@ You should see that you have at least one OSD `up` and `in`. The minimum number (see `storageclass-test.yaml`). In the production storage class example (`storageclass.yaml`), three OSDs would be required. ```console -ceph status -``` +$ ceph status + cluster: + id: a0452c76-30d9-4c1a-a948-5d8405f19a7c + health: HEALTH_OK ->``` -> cluster: -> id: a0452c76-30d9-4c1a-a948-5d8405f19a7c -> health: HEALTH_OK -> -> services: -> mon: 3 daemons, quorum a,b,c (age 11m) -> mgr: a(active, since 10m) -> osd: 1 osds: 1 up (since 46s), 1 in (since 109m) ->``` + services: + mon: 3 daemons, quorum a,b,c (age 11m) + mgr: a(active, since 10m) + osd: 1 osds: 1 up (since 46s), 1 in (since 109m) +``` #### OSD Prepare Logs @@ -285,14 +246,11 @@ If you don't see the expected number of OSDs, let's investigate why they weren't On each node where Rook looks for OSDs to configure, you will see an "osd prepare" pod. ```console -kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare +$ kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare +NAME ... READY STATUS RESTARTS AGE +rook-ceph-osd-prepare-minikube-9twvk 0/2 Completed 0 30m ``` ->``` ->NAME ... READY STATUS RESTARTS AGE ->rook-ceph-osd-prepare-minikube-9twvk 0/2 Completed 0 30m ->``` - See the section on [why OSDs are not getting created](#osd-pods-are-not-created-on-my-devices) to investigate the logs. #### CSI Driver @@ -302,18 +260,17 @@ during the provisioning. There are two provisioner pods: -``` +```console kubectl -n rook-ceph get pod -l app=csi-rbdplugin-provisioner ``` Get the logs of each of the pods. One of them should be the "leader" and be responding to requests. -``` +```console kubectl -n rook-ceph logs csi-cephfsplugin-provisioner-d77bb49c6-q9hwq csi-provisioner ``` -See also the [CSI Troubleshooting Guide](ceph-csi-troubleshooting.md). - +See also the [CSI Troubleshooting Guide](../Troubleshooting/ceph-csi-common-issues.md). #### Operator unresponsiveness @@ -325,7 +282,7 @@ In this case, restart the operator pod to get things going again. ### Solution If the "osd prepare" logs didn't give you enough clues about why the OSDs were not being created, -please review your [cluster.yaml](eph-cluster-crd.html#storage-selection-settings) configuration. +please review your [cluster.yaml](../CRDs/ceph-cluster-crd.md#storage-selection-settings) configuration. The common misconfigurations include: * If `useAllDevices: true`, Rook expects to find local devices attached to the nodes. If no devices are found, no OSDs will be created. @@ -356,17 +313,14 @@ old state. Looking at the OSD pod logs you will see an error about the file alre ```console $ kubectl -n rook-ceph logs rook-ceph-osd-fl8fs +... +2017-10-31 20:13:11.187106 I | mkfs-osd0: 2017-10-31 20:13:11.186992 7f0059d62e00 -1 bluestore(/var/lib/rook/osd0) _read_fsid unparsable uuid +2017-10-31 20:13:11.187208 I | mkfs-osd0: 2017-10-31 20:13:11.187026 7f0059d62e00 -1 bluestore(/var/lib/rook/osd0) _setup_block_symlink_or_file failed to create block symlink to /dev/disk/by-partuuid/651153ba-2dfc-4231-ba06-94759e5ba273: (17) File exists +2017-10-31 20:13:11.187233 I | mkfs-osd0: 2017-10-31 20:13:11.187038 7f0059d62e00 -1 bluestore(/var/lib/rook/osd0) mkfs failed, (17) File exists +2017-10-31 20:13:11.187254 I | mkfs-osd0: 2017-10-31 20:13:11.187042 7f0059d62e00 -1 OSD::mkfs: ObjectStore::mkfs failed with error (17) File exists +2017-10-31 20:13:11.187275 I | mkfs-osd0: 2017-10-31 20:13:11.187121 7f0059d62e00 -1 ** ERROR: error creating empty object store in /var/lib/rook/osd0: (17) File exists ``` ->``` ->... ->2017-10-31 20:13:11.187106 I | mkfs-osd0: 2017-10-31 20:13:11.186992 7f0059d62e00 -1 bluestore(/var/lib/rook/osd0) _read_fsid unparsable uuid ->2017-10-31 20:13:11.187208 I | mkfs-osd0: 2017-10-31 20:13:11.187026 7f0059d62e00 -1 bluestore(/var/lib/rook/osd0) _setup_block_symlink_or_file failed to create block symlink to /dev/disk/by-partuuid/651153ba-2dfc-4231-ba06-94759e5ba273: (17) File exists ->2017-10-31 20:13:11.187233 I | mkfs-osd0: 2017-10-31 20:13:11.187038 7f0059d62e00 -1 bluestore(/var/lib/rook/osd0) mkfs failed, (17) File exists ->2017-10-31 20:13:11.187254 I | mkfs-osd0: 2017-10-31 20:13:11.187042 7f0059d62e00 -1 OSD::mkfs: ObjectStore::mkfs failed with error (17) File exists ->2017-10-31 20:13:11.187275 I | mkfs-osd0: 2017-10-31 20:13:11.187121 7f0059d62e00 -1 ** ERROR: error creating empty object store in /var/lib/rook/osd0: (17) File exists ->``` - ### Solution If the error is from the file that already exists, this is a common problem reinitializing the Rook cluster when the local directory used for persistence has **not** been purged. @@ -385,7 +339,7 @@ Then when the cluster CRD is applied to start a new cluster, the rook-operator s ### Investigation First, ensure that you have specified the devices correctly in the CRD. -The [Cluster CRD](ceph-cluster-crd.md#storage-selection-settings) has several ways to specify the devices that are to be consumed by the Rook storage: +The [Cluster CRD](../CRDs/ceph-cluster-crd.md#storage-selection-settings) has several ways to specify the devices that are to be consumed by the Rook storage: * `useAllDevices: true`: Rook will consume all devices it determines to be available * `deviceFilter`: Consume all devices that match this regular expression @@ -398,50 +352,47 @@ After the job is complete, Rook leaves the pod around in case the logs need to b ```console # Get the prepare pods in the cluster -kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare +$ kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare +NAME READY STATUS RESTARTS AGE +rook-ceph-osd-prepare-node1-fvmrp 0/1 Completed 0 18m +rook-ceph-osd-prepare-node2-w9xv9 0/1 Completed 0 22m +rook-ceph-osd-prepare-node3-7rgnv 0/1 Completed 0 22m ``` ->``` ->NAME READY STATUS RESTARTS AGE ->rook-ceph-osd-prepare-node1-fvmrp 0/1 Completed 0 18m ->rook-ceph-osd-prepare-node2-w9xv9 0/1 Completed 0 22m ->rook-ceph-osd-prepare-node3-7rgnv 0/1 Completed 0 22m ->``` - ```console # view the logs for the node of interest in the "provision" container -kubectl -n rook-ceph logs rook-ceph-osd-prepare-node1-fvmrp provision +$ kubectl -n rook-ceph logs rook-ceph-osd-prepare-node1-fvmrp provision [...] ``` Here are some key lines to look for in the log: ->``` -># A device will be skipped if Rook sees it has partitions or a filesystem ->2019-05-30 19:02:57.353171 W | cephosd: skipping device sda that is in use ->2019-05-30 19:02:57.452168 W | skipping device "sdb5": ["Used by ceph-disk"] -> -># Other messages about a disk being unusable by ceph include: ->Insufficient space (<5GB) on vgs ->Insufficient space (<5GB) ->LVM detected ->Has BlueStore device label ->locked ->read-only -> -># A device is going to be configured ->2019-05-30 19:02:57.535598 I | cephosd: device sdc to be configured by ceph-volume -> -># For each device configured you will see a report printed to the log ->2019-05-30 19:02:59.844642 I | Type Path LV Size % of device ->2019-05-30 19:02:59.844651 I | ---------------------------------------------------------------------------------------------------- ->2019-05-30 19:02:59.844677 I | [data] /dev/sdc 7.00 GB 100% ->``` +```console +# A device will be skipped if Rook sees it has partitions or a filesystem +2019-05-30 19:02:57.353171 W | cephosd: skipping device sda that is in use +2019-05-30 19:02:57.452168 W | skipping device "sdb5": ["Used by ceph-disk"] + +# Other messages about a disk being unusable by ceph include: +Insufficient space (<5GB) on vgs +Insufficient space (<5GB) +LVM detected +Has BlueStore device label +locked +read-only + +# A device is going to be configured +2019-05-30 19:02:57.535598 I | cephosd: device sdc to be configured by ceph-volume + +# For each device configured you will see a report printed to the log +2019-05-30 19:02:59.844642 I | Type Path LV Size % of device +2019-05-30 19:02:59.844651 I | ---------------------------------------------------------------------------------------------------- +2019-05-30 19:02:59.844677 I | [data] /dev/sdc 7.00 GB 100% +``` ### Solution Either update the CR with the correct settings, or clean the partitions or filesystem from your devices. -To clean devices from a previous install see the [cleanup guide](ceph-teardown.md#zapping-devices). +To clean devices from a previous install see the [cleanup guide](../Storage-Configuration/ceph-teardown.md#zapping-devices). After the settings are updated or the devices are cleaned, trigger the operator to analyze the devices again by restarting the operator. Each time the operator starts, it will ensure all the desired devices are configured. The operator does automatically @@ -449,7 +400,7 @@ deploy OSDs in most scenarios, but an operator restart will cover any scenarios ```console # Restart the operator to ensure devices are configured. A new pod will automatically be started when the current operator pod is deleted. -kubectl -n rook-ceph delete pod -l app=rook-ceph-operator +$ kubectl -n rook-ceph delete pod -l app=rook-ceph-operator [...] ``` @@ -468,20 +419,18 @@ On a node running a pod with a Ceph persistent volume ```console mount | grep rbd +# _netdev mount option is absent, also occurs for cephfs +# OS is not aware PV is mounted over network +/dev/rbdx on ... (rw,relatime, ..., noquota) ``` ->``` -># _netdev mount option is absent, also occurs for cephfs -># OS is not aware PV is mounted over network ->/dev/rbdx on ... (rw,relatime, ..., noquota) ->``` When the reboot command is issued, network interfaces are terminated before disks are unmounted. This results in the node hanging as repeated attempts to unmount Ceph persistent volumes fail with the following error: ->``` ->libceph: connect [monitor-ip]:6789 error -101 ->``` +```console +libceph: connect [monitor-ip]:6789 error -101 +``` ### Solution @@ -492,13 +441,13 @@ Because `kubectl drain` command automatically marks the node as unschedulable (` Drain the node: ```console -$ kubectl drain --ignore-daemonsets --delete-local-data +kubectl drain --ignore-daemonsets --delete-local-data ``` Uncordon the node: ```console -$ kubectl uncordon +kubectl uncordon ``` ## Using multiple shared filesystem (CephFS) is attempted on a kernel version older than 4.7 @@ -514,7 +463,7 @@ $ kubectl uncordon The only solution to this problem is to upgrade your kernel to `4.7` or higher. This is due to a mount flag added in the kernel version `4.7` which allows to chose the filesystem by name. -For additional info on the kernel version requirement for multiple shared filesystems (CephFS), see [Filesystem - Kernel version requirement](ceph-filesystem.md#kernel-version-requirement). +For additional info on the kernel version requirement for multiple shared filesystems (CephFS), see [Filesystem - Kernel version requirement](../Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md#kernel-version-requirement). ## Set debug log level for all Ceph daemons @@ -525,18 +474,12 @@ You can find the list of all subsystems and their default values in [Ceph loggin Assuming you want a log level of 1, you will run: ```console -kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level 1 +$ kubectl -n rook-ceph exec deploy/rook-ceph-tools -- set-ceph-debug-level 1 +ceph config set global debug_context 1 +ceph config set global debug_lockdep 1 +[...] ``` -Output: - ->```quote -> ceph config set global debug_context 1 -> ceph config set global debug_lockdep 1 ->... ->... ->``` - Once you are done debugging, you can revert all the debug flag to their default value by running the following: ```console @@ -575,21 +518,19 @@ To disable the logging on file, simply set `log_to_file` to `false`. This happens when the following conditions are satisfied. -- The problematic RBD device and the corresponding OSDs are co-located. -- There is an XFS filesystem on top of this device. +* The problematic RBD device and the corresponding OSDs are co-located. +* There is an XFS filesystem on top of this device. In addition, when this problem happens, you can see the following messages in `dmesg`. ```console -dmesg +$ dmesg +... +[51717.039319] INFO: task kworker/2:1:5938 blocked for more than 120 seconds. +[51717.039361] Not tainted 4.15.0-72-generic #81-Ubuntu +[51717.039388] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. +... ``` ->``` ->... ->[51717.039319] INFO: task kworker/2:1:5938 blocked for more than 120 seconds. ->[51717.039361] Not tainted 4.15.0-72-generic #81-Ubuntu ->[51717.039388] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. ->... ->``` It's so-called `hung_task` problem and means that there is a deadlock in the kernel. For more detail, please refer to [the corresponding issue comment](https://github.com/rook/rook/issues/3132#issuecomment-580508760). @@ -606,24 +547,22 @@ You can bypass this problem by using ext4 or any other filesystems rather than X ### Symptoms -- `ceph status` shows "too few PGs per OSD" warning as follows. +* `ceph status` shows "too few PGs per OSD" warning as follows. ```console -ceph status +$ ceph status + cluster: + id: fd06d7c3-5c5c-45ca-bdea-1cf26b783065 + health: HEALTH_WARN + too few PGs per OSD (16 < min 30) +[...] ``` ->``` -> cluster: -> id: fd06d7c3-5c5c-45ca-bdea-1cf26b783065 -> health: HEALTH_WARN -> too few PGs per OSD (16 < min 30) ->``` - ### Solution The meaning of this warning is written in [the document](https://docs.ceph.com/docs/master/rados/operations/health-checks#too-few-pgs). However, in many cases it is benign. For more information, please see [the blog entry](http://ceph.com/community/new-luminous-pg-overdose-protection/). -Please refer to [Configuring Pools](ceph-advanced-configuration.md#configuring-pools) if you want to know the proper `pg_num` of pools and change these values. +Please refer to [Configuring Pools](../Storage-Configuration/Advanced/ceph-configuration.md#configuring-pools) if you want to know the proper `pg_num` of pools and change these values. ## LVM metadata can be corrupted with OSD on LV-backed PVC @@ -635,13 +574,13 @@ If you still decide to configure an OSD on LVM, please keep the following in min ### Solution -- Disable `lvmetad.` -- Avoid configuration of LVs from the host. In addition, don't touch the VGs and physical volumes that back these LVs. -- Avoid incrementing the `count` field of `storageClassDeviceSets` and create a new LV that backs an OSD simultaneously. +* Disable `lvmetad.` +* Avoid configuration of LVs from the host. In addition, don't touch the VGs and physical volumes that back these LVs. +* Avoid incrementing the `count` field of `storageClassDeviceSets` and create a new LV that backs an OSD simultaneously. -You can know whether the above-mentioned tag exists with the command: `sudo lvs -o lv_name,lv_tags`. If the `lv_tag` field is empty in an LV corresponding to the OSD lv_tags, this OSD encountered the problem. In this case, please [retire this OSD](ceph-osd-mgmt.md#remove-an-osd) or replace with other new OSD before restarting. +You can know whether the above-mentioned tag exists with the command: `sudo lvs -o lv_name,lv_tags`. If the `lv_tag` field is empty in an LV corresponding to the OSD lv_tags, this OSD encountered the problem. In this case, please [retire this OSD](../Storage-Configuration/Advanced/ceph-osd-mgmt.md#remove-an-osd) or replace with other new OSD before restarting. -This problem doesn't happen in newly created LV-backed PVCs because OSD container doesn't modify LVM metadata anymore. The existing lvm mode OSDs work continuously even thought upgrade your Rook. However, using the raw mode OSDs is recommended because of the above-mentioned problem. You can replace the existing OSDs with raw mode OSDs by retiring them and adding new OSDs one by one. See the documents [Remove an OSD](ceph-osd-mgmt.md#remove-an-osd) and [Add an OSD on a PVC](ceph-osd-mgmt.md#add-an-osd-on-a-pvc). +This problem doesn't happen in newly created LV-backed PVCs because OSD container doesn't modify LVM metadata anymore. The existing lvm mode OSDs work continuously even thought upgrade your Rook. However, using the raw mode OSDs is recommended because of the above-mentioned problem. You can replace the existing OSDs with raw mode OSDs by retiring them and adding new OSDs one by one. See the documents [Remove an OSD](../Storage-Configuration/Advanced/ceph-osd-mgmt.md#remove-an-osd) and [Add an OSD on a PVC](../Storage-Configuration/Advanced/ceph-osd-mgmt.md#add-an-osd-on-a-pvc). ## OSD prepare job fails due to low aio-max-nr setting @@ -659,6 +598,7 @@ Alternatively, you can have a [DaemonSet](https://github.com/rook/rook/issues/62 ## Unexpected partitions created ### Symptoms + **Users running Rook versions v1.6.0-v1.6.7 may observe unwanted OSDs on partitions that appear unexpectedly and seemingly randomly, which can corrupt existing OSDs.** @@ -677,7 +617,8 @@ Below is an example of `lsblk` output from a node where phantom Atari partitions that `sdX1` is never present for the phantom partitions, and `sdX2` is 48G on all disks. `sdX3` is a variable size and may not always be present. It is possible for `sdX4` to appear, though it is an anecdotally rare event. -``` + +```console # lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sdb 8:16 0 3T 0 disk @@ -691,10 +632,12 @@ sdd 8:48 0 3T 0 disk └─sdd3 8:51 0 6.3M 0 part ``` -You can see https://github.com/rook/rook/issues/7940 for more detailed information and discussion. +You can see [GitHub rook/rook - Issue 7940 unexpected partition on disks >= 1TB (atari partitions)](https://github.com/rook/rook/issues/7940) for more detailed information and discussion. ### Solution + #### Recover from corruption (v1.6.0-v1.6.7) + If you are using Rook v1.6, you must first update to v1.6.8 or higher to avoid further incidents of OSD corruption caused by these Atari partitions. @@ -705,20 +648,21 @@ workaround are still at risk for OSD failures in the future. To resolve the issue, immediately update to v1.6.8 or higher. After the update, no corruption should occur on OSDs created in the future. Next, to get back to a healthy Ceph cluster state, focus on one -corruped disk at a time and [remove all OSDs on each corrupted disk](ceph-osd-mgmt.md#remove-an-osd) +corruped disk at a time and [remove all OSDs on each corrupted disk](../Storage-Configuration/Advanced/ceph-osd-mgmt.md#remove-an-osd) one disk at a time. As an example, you may have `/dev/sdb` with two unexpected partitions (`/dev/sdb2` and `/dev/sdb3`) as well as a second corrupted disk `/dev/sde` with one unexpected partition (`/dev/sde2`). + 1. First, remove the OSDs associated with `/dev/sdb`, `/dev/sdb2`, and `/dev/sdb3`. There might be only one, or up to 3 OSDs depending on how your system was affected. Again see the - [OSD management doc](ceph-osd-mgmt.md#remove-an-osd). + [OSD management doc](../Storage-Configuration/Advanced/ceph-osd-mgmt.md#remove-an-osd). 2. Use `dd` to wipe the first sectors of the partitions followed by the disk itself. E.g., - - `dd if=/dev/zero of=/dev/sdb2 bs=1M` - - `dd if=/dev/zero of=/dev/sdb3 bs=1M` - - `dd if=/dev/zero of=/dev/sdb bs=1M` + * `dd if=/dev/zero of=/dev/sdb2 bs=1M` + * `dd if=/dev/zero of=/dev/sdb3 bs=1M` + * `dd if=/dev/zero of=/dev/sdb bs=1M` 3. Then wipe clean `/dev/sdb` to prepare it for a new OSD. - See [the teardown document](ceph-teardown.md#zapping-devices) for details. + See [the teardown document](../Storage-Configuration/ceph-teardown.md#zapping-devices) for details. 4. After this, scale up the Rook operator to deploy a new OSD to `/dev/sdb`. This will allow Ceph to use `/dev/sdb` for data recovery and replication while the next OSDs are removed. 5. Now Repeat steps 1-4 for `/dev/sde` and `/dev/sde2`, and continue for any other corruped disks. @@ -736,7 +680,7 @@ the discover daemonset is not created, even though `ROOK_ENABLE_DISCOVERY_DAEMON ### Investigation Inspect the `rook-ceph-operator-config` ConfigMap for conflicting settings. The ConfigMap takes -precedence over the environment. The ConfigMap [must exist](ceph-advanced-configuration.md#configuration-using-environment-variables), +precedence over the environment. The ConfigMap [must exist](../Storage-Configuration/Advanced/ceph-configuration.md#configuration-using-environment-variables), even if all actual configuration is supplied through the environment. Look for lines with the `op-k8sutil` prefix in the operator logs. diff --git a/Documentation/ceph-csi-troubleshooting.md b/Documentation/Troubleshooting/ceph-csi-common-issues.md similarity index 74% rename from Documentation/ceph-csi-troubleshooting.md rename to Documentation/Troubleshooting/ceph-csi-common-issues.md index 34507c47f8e1..f18b2c31dbe8 100644 --- a/Documentation/ceph-csi-troubleshooting.md +++ b/Documentation/Troubleshooting/ceph-csi-common-issues.md @@ -1,18 +1,14 @@ --- title: CSI Common Issues -weight: 11125 -indent: true --- -# CSI Common Issues - Issues when provisioning volumes with the Ceph CSI driver can happen for many reasons such as: -- Network connectivity between CSI pods and ceph -- Cluster health issues -- Slow operations -- Kubernetes issues -- Ceph-CSI configuration or bugs +* Network connectivity between CSI pods and ceph +* Cluster health issues +* Slow operations +* Kubernetes issues +* Ceph-CSI configuration or bugs The following troubleshooting steps can help identify a number of issues. @@ -32,16 +28,13 @@ The Ceph monitors are the most critical component of the cluster to check first. Retrieve the mon endpoints from the services: ```console -kubectl -n rook-ceph get svc -l app=rook-ceph-mon +$ kubectl -n rook-ceph get svc -l app=rook-ceph-mon +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rook-ceph-mon-a ClusterIP 10.104.165.31 6789/TCP,3300/TCP 18h +rook-ceph-mon-b ClusterIP 10.97.244.93 6789/TCP,3300/TCP 21s +rook-ceph-mon-c ClusterIP 10.99.248.163 6789/TCP,3300/TCP 8s ``` ->``` ->NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ->rook-ceph-mon-a ClusterIP 10.104.165.31 6789/TCP,3300/TCP 18h ->rook-ceph-mon-b ClusterIP 10.97.244.93 6789/TCP,3300/TCP 21s ->rook-ceph-mon-c ClusterIP 10.99.248.163 6789/TCP,3300/TCP 8s ->``` - If host networking is enabled in the CephCluster CR, you will instead need to find the node IPs for the hosts where the mons are running. @@ -50,8 +43,8 @@ These endpoints must be accessible by all clients in the cluster, including the If you are seeing issues provisioning the PVC then you need to check the network connectivity from the provisioner pods. -- For CephFS PVCs, check network connectivity from the `csi-cephfsplugin` container of the `csi-cephfsplugin-provisioner` pods -- For Block PVCs, check network connectivity from the `csi-rbdplugin` container of the `csi-rbdplugin-provisioner` pods +* For CephFS PVCs, check network connectivity from the `csi-cephfsplugin` container of the `csi-cephfsplugin-provisioner` pods +* For Block PVCs, check network connectivity from the `csi-rbdplugin` container of the `csi-rbdplugin-provisioner` pods For redundancy, there are two provisioner pods for each type. Make sure to test connectivity from all provisioner pods. @@ -91,21 +84,20 @@ Even slow ops in the ceph cluster can contribute to the issues. In the toolbox, make sure that no slow ops are present and the ceph cluster is healthy ```console -ceph -s +$ ceph -s +cluster: + id: ba41ac93-3b55-4f32-9e06-d3d8c6ff7334 + health: HEALTH_WARN + 30 slow ops, oldest one blocked for 10624 sec, mon.a has slow ops +[...] ``` ->``` ->cluster: -> id: ba41ac93-3b55-4f32-9e06-d3d8c6ff7334 -> health: HEALTH_WARN -> 30 slow ops, oldest one blocked for 10624 sec, mon.a has slow ops ->``` If Ceph is not healthy, check the following health for more clues: -- The Ceph monitor logs for errors -- The OSD logs for errors -- Disk Health -- Network Health +* The Ceph monitor logs for errors +* The OSD logs for errors +* Disk Health +* Network Health ## Ceph Troubleshooting @@ -117,14 +109,11 @@ Suppose the pool name mentioned in the `storageclass.yaml` is `replicapool`. It to exist in the toolbox: ```console -ceph osd lspools +$ ceph osd lspools +1 device_health_metrics +2 replicapool ``` ->``` ->1 device_health_metrics ->2 replicapool ->``` - If the pool is not in the list, create the `CephBlockPool` CR for the pool if you have not already. If you have already created the pool, check the Rook operator log for errors creating the pool. @@ -135,25 +124,20 @@ For the shared filesystem (CephFS), check that the filesystem and pools you have Suppose the `fsName` name mentioned in the `storageclass.yaml` is `myfs`. It can be verified in the toolbox: ```console -ceph fs ls +$ ceph fs ls +name: myfs, metadata pool: myfs-metadata, data pools: [myfs-data0 ] ``` ->``` ->name: myfs, metadata pool: myfs-metadata, data pools: [myfs-data0 ] ->``` Now verify the `pool` mentioned in the `storageclass.yaml` exists, such as the example `myfs-data0`. ```console ceph osd lspools +1 device_health_metrics +2 replicapool +3 myfs-metadata0 +4 myfs-data0 ``` ->``` ->1 device_health_metrics ->2 replicapool ->3 myfs-metadata0 ->4 myfs-data0 ->``` - The pool for the filesystem will have the suffix `-data0` compared the filesystem name that is created by the CephFilesystem CR. @@ -164,17 +148,14 @@ Ceph-CSI creates the default subvolumegroup with the name csi. Verify that the s exists: ```console -ceph fs subvolumegroup ls myfs +$ ceph fs subvolumegroup ls myfs +[ + { + "name": "csi" + } +] ``` ->``` ->[ -> { -> "name": "csi" -> } ->] ->``` - If you don’t see any issues with your Ceph cluster, the following sections will start debugging the issue from the CSI side. ## Provisioning Volumes @@ -221,18 +202,15 @@ be found [here](https://github.com/kubernetes-csi/external-snapshotter). **In Kubernetes 1.17 the volume snapshot feature was promoted to beta. In Kubernetes 1.20, the feature gate is enabled by default on standard Kubernetes deployments and cannot be turned off.** Make sure you have installed the correct snapshotter CRD version. If you have not installed the snapshotter -controller, see the [Snapshots guide](ceph-csi-snapshot.md). +controller, see the [Snapshots guide](../Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md). ```console -kubectl get crd | grep snapshot +$ kubectl get crd | grep snapshot +volumesnapshotclasses.snapshot.storage.k8s.io 2021-01-25T11:19:38Z +volumesnapshotcontents.snapshot.storage.k8s.io 2021-01-25T11:19:39Z +volumesnapshots.snapshot.storage.k8s.io 2021-01-25T11:19:40Z ``` ->``` ->volumesnapshotclasses.snapshot.storage.k8s.io 2021-01-25T11:19:38Z ->volumesnapshotcontents.snapshot.storage.k8s.io 2021-01-25T11:19:39Z ->volumesnapshots.snapshot.storage.k8s.io 2021-01-25T11:19:40Z ->``` - The above CRDs must have the matching version in your `snapshotclass.yaml` or `snapshot.yaml`. Otherwise, the `VolumeSnapshot` and `VolumesnapshotContent` will not be created. @@ -254,10 +232,10 @@ kubectl -n rook-ceph logs deploy/csi-rbdplugin-provisioner -c csi-snapshotter If you see an error such as: ->``` ->GRPC error: rpc error: code = Aborted desc = an operation with the given Volume ID ->0001-0009-rook-ceph-0000000000000001-8d0ba728-0e17-11eb-a680-ce6eecc894de already >exists. ->``` +```console +GRPC error: rpc error: code = Aborted desc = an operation with the given Volume ID +0001-0009-rook-ceph-0000000000000001-8d0ba728-0e17-11eb-a680-ce6eecc894de already >exists. +``` The issue typically is in the Ceph cluster or network connectivity. If the issue is in Provisioning the PVC Restarting the Provisioner pods help(for CephFS issue @@ -270,9 +248,9 @@ for CephFS issue. When a user requests to create the application pod with PVC, there is a three-step process -- CSI driver registration -- Create volume attachment object -- Stage and publish the volume +* CSI driver registration +* Create volume attachment object +* Stage and publish the volume ### csi-driver registration @@ -292,21 +270,20 @@ If any issue exists in attaching the PVC to the application pod check logs from sidecar container in plugin pod where your application pod is scheduled. ```console -kubectl -n rook-ceph logs deploy/csi-rbdplugin -c driver-registrar +$ kubectl -n rook-ceph logs deploy/csi-rbdplugin -c driver-registrar +[...] +I0120 12:28:34.231761 124018 main.go:112] Version: v2.0.1 +I0120 12:28:34.233910 124018 connection.go:151] Connecting to unix:///csi/csi.sock +I0120 12:28:35.242469 124018 node_register.go:55] Starting Registration Server at: /registration/rook-ceph.rbd.csi.ceph.com-reg.sock +I0120 12:28:35.243364 124018 node_register.go:64] Registration Server started at: /registration/rook-ceph.rbd.csi.ceph.com-reg.sock +I0120 12:28:35.243673 124018 node_register.go:86] Skipping healthz server because port set to: 0 +I0120 12:28:36.318482 124018 main.go:79] Received GetInfo call: &InfoRequest{} +I0120 12:28:37.455211 124018 main.go:89] Received NotifyRegistrationStatus call: &RegistrationStatus{PluginRegistered:true,Error:,} +E0121 05:19:28.658390 124018 connection.go:129] Lost connection to unix:///csi/csi.sock. +E0125 07:11:42.926133 124018 connection.go:129] Lost connection to unix:///csi/csi.sock. +[...] ``` ->``` ->I0120 12:28:34.231761 124018 main.go:112] Version: v2.0.1 ->I0120 12:28:34.233910 124018 connection.go:151] Connecting to unix:///csi/csi.sock ->I0120 12:28:35.242469 124018 node_register.go:55] Starting Registration Server at: /registration/rook-ceph.rbd.csi.ceph.com-reg.sock ->I0120 12:28:35.243364 124018 node_register.go:64] Registration Server started at: /registration/rook-ceph.rbd.csi.ceph.com-reg.sock ->I0120 12:28:35.243673 124018 node_register.go:86] Skipping healthz server because port set to: 0 ->I0120 12:28:36.318482 124018 main.go:79] Received GetInfo call: &InfoRequest{} ->I0120 12:28:37.455211 124018 main.go:89] Received NotifyRegistrationStatus call: &RegistrationStatus{PluginRegistered:true,Error:,} ->E0121 05:19:28.658390 124018 connection.go:129] Lost connection to unix:///csi/csi.sock. ->E0125 07:11:42.926133 124018 connection.go:129] Lost connection to unix:///csi/csi.sock. ->``` - You should see the response `RegistrationStatus{PluginRegistered:true,Error:,}` in the logs to confirm that plugin is registered with kubelet. @@ -328,14 +305,11 @@ If any issue exists in attaching the PVC to the application pod first check the and also log from csi-attacher sidecar container in provisioner pod. ```console -kubectl get volumeattachment +$ kubectl get volumeattachment +NAME ATTACHER PV NODE ATTACHED AGE +csi-75903d8a902744853900d188f12137ea1cafb6c6f922ebc1c116fd58e950fc92 rook-ceph.cephfs.csi.ceph.com pvc-5c547d2a-fdb8-4cb2-b7fe-e0f30b88d454 minikube true 4m26s ``` ->``` ->NAME ATTACHER PV NODE ATTACHED AGE ->csi-75903d8a902744853900d188f12137ea1cafb6c6f922ebc1c116fd58e950fc92 rook-ceph.cephfs.csi.ceph.com pvc-5c547d2a-fdb8-4cb2-b7fe-e0f30b88d454 minikube true 4m26s ->``` - ```console kubectl logs po/csi-rbdplugin-provisioner-d857bfb5f-ddctl -c csi-attacher ``` @@ -350,15 +324,15 @@ Identify the `csi-cephfsplugin-xxxx` pod running on the node where your applicat `kubectl get po -o wide` and match the node names. ```console -kubectl exec -it csi-cephfsplugin-tfk2g -c csi-cephfsplugin -- sh -ps -ef |grep mount - +$ kubectl exec -it csi-cephfsplugin-tfk2g -c csi-cephfsplugin -- sh +$ ps -ef |grep mount +[...] root 67 60 0 11:55 pts/0 00:00:00 grep mount ``` ```console ps -ef |grep ceph - +[...] root 1 0 0 Jan20 ? 00:00:26 /usr/local/bin/cephcsi --nodeid=minikube --type=cephfs --endpoint=unix:///csi/csi.sock --v=0 --nodeserver=true --drivername=rook-ceph.cephfs.csi.ceph.com --pidlimit=-1 --metricsport=9091 --forcecephkernelclient=true --metricspath=/metrics --enablegrpcmetrics=true root 69 60 0 11:55 pts/0 00:00:00 grep ceph ``` @@ -378,44 +352,37 @@ Identify the `csi-rbdplugin-xxxx` pod running on the node where your application `kubectl get po -o wide` and match the node names. ```console -kubectl exec -it csi-rbdplugin-vh8d5 -c csi-rbdplugin -- sh -``` -```console -ps -ef |grep map +$ kubectl exec -it csi-rbdplugin-vh8d5 -c csi-rbdplugin -- sh +$ ps -ef |grep map +[...] +root 1297024 1296907 0 12:00 pts/0 00:00:00 grep map ``` ->``` ->root 1297024 1296907 0 12:00 pts/0 00:00:00 grep map ->``` ```console -ps -ef |grep mount +$ ps -ef |grep mount +[...] +root 1824 1 0 Jan19 ? 00:00:00 /usr/sbin/rpc.mountd +ceph 1041020 1040955 1 07:11 ? 00:03:43 ceph-mgr --fsid=ba41ac93-3b55-4f32-9e06-d3d8c6ff7334 --keyring=/etc/ceph/keyring-store/keyring --log-to-stderr=true --err-to-stderr=true --mon-cluster-log-to-stderr=true --log-stderr-prefix=debug --default-log-to-file=false --default-mon-cluster-log-to-file=false --mon-host=[v2:10.111.136.166:3300,v1:10.111.136.166:6789] --mon-initial-members=a --id=a --setuser=ceph --setgroup=ceph --client-mount-uid=0 --client-mount-gid=0 --foreground --public-addr=172.17.0.6 +root 1297115 1296907 0 12:00 pts/0 00:00:00 grep mount ``` ->``` ->root 1824 1 0 Jan19 ? 00:00:00 /usr/sbin/rpc.mountd ->ceph 1041020 1040955 1 07:11 ? 00:03:43 ceph-mgr --fsid=ba41ac93-3b55-4f32-9e06-d3d8c6ff7334 --keyring=/etc/ceph/keyring-store/keyring --log-to-stderr=true --err-to-stderr=true --mon-cluster-log-to-stderr=true --log-stderr-prefix=debug --default-log-to-file=false --default-mon-cluster-log-to-file=false --mon-host=[v2:10.111.136.166:3300,v1:10.111.136.166:6789] --mon-initial-members=a --id=a --setuser=ceph --setgroup=ceph --client-mount-uid=0 --client-mount-gid=0 --foreground --public-addr=172.17.0.6 ->root 1297115 1296907 0 12:00 pts/0 00:00:00 grep mount ->``` ```console -ps -ef |grep mkfs +$ ps -ef |grep mkfs +[...] +root 1297291 1296907 0 12:00 pts/0 00:00:00 grep mkfs ``` ->``` ->root 1297291 1296907 0 12:00 pts/0 00:00:00 grep mkfs ->``` ```console -ps -ef |grep umount +$ ps -ef |grep umount +[...] +root 1298500 1296907 0 12:01 pts/0 00:00:00 grep umount ``` ->``` ->root 1298500 1296907 0 12:01 pts/0 00:00:00 grep umount ->``` ```console -ps -ef |grep unmap +$ ps -ef |grep unmap +[...] +root 1298578 1296907 0 12:01 pts/0 00:00:00 grep unmap ``` ->``` ->root 1298578 1296907 0 12:01 pts/0 00:00:00 grep unmap ->``` If any commands are stuck check the **dmesg** logs from the node. Restarting the `csi-rbdplugin` pod also may help sometimes. @@ -437,7 +404,7 @@ If nothing else helps, get the last executed command from the ceph-csi pod logs the provisioner or plugin pod to see if there are errors returned even if they couldn't be seen in the logs. ```console -$ rbd ls --id=csi-rbd-node -m=10.111.136.166:6789 --key=AQDpIQhg+v83EhAAgLboWIbl+FL/nThJzoI3Fg== +rbd ls --id=csi-rbd-node -m=10.111.136.166:6789 --key=AQDpIQhg+v83EhAAgLboWIbl+FL/nThJzoI3Fg== ``` Where `-m` is one of the mon endpoints and the `--key` is the key used by the CSI driver for accessing the Ceph cluster. @@ -448,12 +415,12 @@ When a node is lost, you will see application pods on the node stuck in the `Ter To allow the application pod to start on another node, force delete the pod. -### Force deleting the pod +### Force deleting the pod To force delete the pod stuck in the `Terminating` state: ```console -$ kubectl -n rook-ceph delete pod my-app-69cd495f9b-nl6hf --grace-period 0 --force +kubectl -n rook-ceph delete pod my-app-69cd495f9b-nl6hf --grace-period 0 --force ``` After the force delete, wait for a timeout of about 8-10 minutes. If the pod still not in the running state, continue with the next section to blocklist the node. diff --git a/Documentation/ceph-toolbox.md b/Documentation/Troubleshooting/ceph-toolbox.md similarity index 91% rename from Documentation/ceph-toolbox.md rename to Documentation/Troubleshooting/ceph-toolbox.md index c360331aeaa3..cde8e4f66969 100644 --- a/Documentation/ceph-toolbox.md +++ b/Documentation/Troubleshooting/ceph-toolbox.md @@ -1,19 +1,17 @@ --- title: Toolbox -weight: 11100 -indent: true --- -# Rook Toolbox - The Rook toolbox is a container with common tools used for rook debugging and testing. The toolbox is based on CentOS, so more tools of your choosing can be easily installed with `yum`. The toolbox can be run in two modes: + 1. [Interactive](#interactive-toolbox): Start a toolbox pod where you can connect and execute Ceph commands from a shell 2. [One-time job](#toolbox-job): Run a script with Ceph commands and collect the results from the job log -> Prerequisite: Before running the toolbox you should have a running Rook cluster deployed (see the [Quickstart Guide](quickstart.md)). +!!! hint + Before running the toolbox you should have a running Rook cluster deployed (see the [Quickstart Guide](../Getting-Started/quickstart.md)). ## Interactive Toolbox diff --git a/Documentation/common-issues.md b/Documentation/Troubleshooting/common-issues.md similarity index 96% rename from Documentation/common-issues.md rename to Documentation/Troubleshooting/common-issues.md index ef2f4c66e3a4..bab9ea11bc50 100644 --- a/Documentation/common-issues.md +++ b/Documentation/Troubleshooting/common-issues.md @@ -1,18 +1,15 @@ --- title: Common Issues -weight: 10500 --- -# Common Issues - To help troubleshoot your Rook clusters, here are some tips on what information will help solve the issues you might be seeing. If after trying the suggestions found on this page and the problem is not resolved, the Rook team is very happy to help you troubleshoot the issues in their Slack channel. Once you have [registered for the Rook Slack](https://slack.rook.io), proceed to the General channel to ask for assistance. -## Ceph +## Ceph Common Issues For common issues specific to Ceph, see the [Ceph Common Issues](ceph-common-issues.md) page. -# Troubleshooting Techniques +## Troubleshooting Techniques Kubernetes status and logs are the the main resources needed to investigate issues in any Rook cluster. diff --git a/Documentation/direct-tools.md b/Documentation/Troubleshooting/direct-tools.md similarity index 93% rename from Documentation/direct-tools.md rename to Documentation/Troubleshooting/direct-tools.md index f3478c976702..1ff3a92c2872 100644 --- a/Documentation/direct-tools.md +++ b/Documentation/Troubleshooting/direct-tools.md @@ -1,11 +1,7 @@ --- title: Direct Tools -weight: 11200 -indent: true --- -# Direct Tools - Rook is designed with Kubernetes design principles from the ground up. This topic is going to escape the bounds of Kubernetes storage and show you how to use block and file storage directly from a pod without any of the Kubernetes magic. The purpose of this topic is to help you quickly test a new configuration, although it is not meant to be used in production. All of the benefits of Kubernetes storage including failover, detach, and attach will not be available. @@ -28,7 +24,7 @@ $ kubectl -n rook-ceph exec -it bash ## Block Storage Tools -After you have created a pool as described in the [Block Storage](ceph-block.md) topic, you can create a block image and mount it directly in a pod. +After you have created a pool as described in the [Block Storage](../Storage-Configuration/Block-Storage-RBD/block-storage.md) topic, you can create a block image and mount it directly in a pod. This example will show how the Ceph rbd volume can be mounted in the direct mount pod. Create the [Direct Mount Pod](direct-tools.md#Start-the-Direct-Mount-Pod). @@ -79,7 +75,7 @@ rbd unmap /dev/rbd0 ## Shared Filesystem Tools -After you have created a filesystem as described in the [Shared Filesystem](ceph-filesystem.md) topic, you can mount the filesystem from multiple pods. +After you have created a filesystem as described in the [Shared Filesystem](../Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md) topic, you can mount the filesystem from multiple pods. The the other topic you may have mounted the filesystem already in the registry pod. Now we will mount the same filesystem in the Direct Mount pod. This is just a simple way to validate the Ceph filesystem and is not recommended for production Kubernetes pods. diff --git a/Documentation/ceph-disaster-recovery.md b/Documentation/Troubleshooting/disaster-recovery.md similarity index 84% rename from Documentation/ceph-disaster-recovery.md rename to Documentation/Troubleshooting/disaster-recovery.md index 9e884ecc7fb9..7968331654ad 100644 --- a/Documentation/ceph-disaster-recovery.md +++ b/Documentation/Troubleshooting/disaster-recovery.md @@ -1,16 +1,8 @@ --- title: Disaster Recovery -weight: 11600 -indent: true --- -# Disaster Recovery - -Under extenuating circumstances, steps may be necessary to recover the cluster health. There are several types of recovery addressed in this document: -* [Restoring Mon Quorum](#restoring-mon-quorum) -* [Restoring CRDs After Deletion](#restoring-crds-after-deletion) -* [Adopt an existing Rook Ceph cluster into a new Kubernetes cluster](#adopt-an-existing-rook-ceph-cluster-into-a-new-kubernetes-cluster) -* [Backing up and restoring a cluster based on PVCs into a new Kubernetes cluster](#backing-up-and-restoring-a-cluster-based-on-pvcs-into-a-new-kubernetes-cluster) +Under extenuating circumstances, steps may be necessary to recover the cluster health. There are several types of recovery addressed in this document. ## Restoring Mon Quorum @@ -32,7 +24,8 @@ kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas=0 ### Inject a new monmap -> **WARNING**: Injecting a monmap must be done very carefully. If run incorrectly, your cluster could be permanently destroyed. +!!! warning + Injecting a monmap must be done very carefully. If run incorrectly, your cluster could be permanently destroyed. The Ceph monmap keeps track of the mon quorum. We will update the monmap to only contain the healthy mon. In this example, the healthy mon is `rook-ceph-mon-b`, while the unhealthy mons are `rook-ceph-mon-a` and `rook-ceph-mon-c`. @@ -111,70 +104,68 @@ kubectl -n rook-ceph patch deployment rook-ceph-mon-b -p '{"spec": {"template": Connect to the pod of a healthy mon and run the following commands. ```console -kubectl -n rook-ceph exec -it bash +$ kubectl -n rook-ceph exec -it bash +# set a few simple variables +cluster_namespace=rook-ceph +good_mon_id=b +monmap_path=/tmp/monmap + +# extract the monmap to a file, by pasting the ceph mon command +# from the good mon deployment and adding the +# `--extract-monmap=${monmap_path}` flag +ceph-mon \ + --fsid=41a537f2-f282-428e-989f-a9e07be32e47 \ + --keyring=/etc/ceph/keyring-store/keyring \ + --log-to-stderr=true \ + --err-to-stderr=true \ + --mon-cluster-log-to-stderr=true \ + --log-stderr-prefix=debug \ + --default-log-to-file=false \ + --default-mon-cluster-log-to-file=false \ + --mon-host=$ROOK_CEPH_MON_HOST \ + --mon-initial-members=$ROOK_CEPH_MON_INITIAL_MEMBERS \ + --id=b \ + --setuser=ceph \ + --setgroup=ceph \ + --foreground \ + --public-addr=10.100.13.242 \ + --setuser-match-path=/var/lib/ceph/mon/ceph-b/store.db \ + --public-bind-addr=$ROOK_POD_IP \ + --extract-monmap=${monmap_path} + +# review the contents of the monmap +monmaptool --print /tmp/monmap + +# remove the bad mon(s) from the monmap +monmaptool ${monmap_path} --rm + +# in this example we remove mon0 and mon2: +monmaptool ${monmap_path} --rm a +monmaptool ${monmap_path} --rm c + +# inject the modified monmap into the good mon, by pasting +# the ceph mon command and adding the +# `--inject-monmap=${monmap_path}` flag, like this +ceph-mon \ + --fsid=41a537f2-f282-428e-989f-a9e07be32e47 \ + --keyring=/etc/ceph/keyring-store/keyring \ + --log-to-stderr=true \ + --err-to-stderr=true \ + --mon-cluster-log-to-stderr=true \ + --log-stderr-prefix=debug \ + --default-log-to-file=false \ + --default-mon-cluster-log-to-file=false \ + --mon-host=$ROOK_CEPH_MON_HOST \ + --mon-initial-members=$ROOK_CEPH_MON_INITIAL_MEMBERS \ + --id=b \ + --setuser=ceph \ + --setgroup=ceph \ + --foreground \ + --public-addr=10.100.13.242 \ + --setuser-match-path=/var/lib/ceph/mon/ceph-b/store.db \ + --public-bind-addr=$ROOK_POD_IP \ + --inject-monmap=${monmap_path} ``` ->``` -># set a few simple variables ->cluster_namespace=rook-ceph ->good_mon_id=b ->monmap_path=/tmp/monmap -> -># extract the monmap to a file, by pasting the ceph mon command -># from the good mon deployment and adding the -># `--extract-monmap=${monmap_path}` flag ->ceph-mon \ -> --fsid=41a537f2-f282-428e-989f-a9e07be32e47 \ -> --keyring=/etc/ceph/keyring-store/keyring \ -> --log-to-stderr=true \ -> --err-to-stderr=true \ -> --mon-cluster-log-to-stderr=true \ -> --log-stderr-prefix=debug \ -> --default-log-to-file=false \ -> --default-mon-cluster-log-to-file=false \ -> --mon-host=$ROOK_CEPH_MON_HOST \ -> --mon-initial-members=$ROOK_CEPH_MON_INITIAL_MEMBERS \ -> --id=b \ -> --setuser=ceph \ -> --setgroup=ceph \ -> --foreground \ -> --public-addr=10.100.13.242 \ -> --setuser-match-path=/var/lib/ceph/mon/ceph-b/store.db \ -> --public-bind-addr=$ROOK_POD_IP \ -> --extract-monmap=${monmap_path} -> -># review the contents of the monmap ->monmaptool --print /tmp/monmap -> -># remove the bad mon(s) from the monmap ->monmaptool ${monmap_path} --rm -> -># in this example we remove mon0 and mon2: ->monmaptool ${monmap_path} --rm a ->monmaptool ${monmap_path} --rm c -> -># inject the modified monmap into the good mon, by pasting -># the ceph mon command and adding the -># `--inject-monmap=${monmap_path}` flag, like this ->ceph-mon \ -> --fsid=41a537f2-f282-428e-989f-a9e07be32e47 \ -> --keyring=/etc/ceph/keyring-store/keyring \ -> --log-to-stderr=true \ -> --err-to-stderr=true \ -> --mon-cluster-log-to-stderr=true \ -> --log-stderr-prefix=debug \ -> --default-log-to-file=false \ -> --default-mon-cluster-log-to-file=false \ -> --mon-host=$ROOK_CEPH_MON_HOST \ -> --mon-initial-members=$ROOK_CEPH_MON_INITIAL_MEMBERS \ -> --id=b \ -> --setuser=ceph \ -> --setgroup=ceph \ -> --foreground \ -> --public-addr=10.100.13.242 \ -> --setuser-match-path=/var/lib/ceph/mon/ceph-b/store.db \ -> --public-bind-addr=$ROOK_POD_IP \ -> --inject-monmap=${monmap_path} ->``` Exit the shell to continue. @@ -208,7 +199,8 @@ mon_host=$(kubectl -n rook-ceph get svc rook-ceph-mon-b -o jsonpath='{.spec.clus kubectl -n rook-ceph patch secret rook-ceph-config -p '{"stringData": {"mon_host": "[v2:'"${mon_host}"':3300,v1:'"${mon_host}"':6789]", "mon_initial_members": "'"${good_mon_id}"'"}}' ``` -> **NOTE**: If you are using `hostNetwork: true`, you need to replace the `mon_host` var with the node IP the mon is pinned to (`nodeSelector`). This is because there is no `rook-ceph-mon-*` service created in that "mode". +!!! note + If you are using `hostNetwork: true`, you need to replace the `mon_host` var with the node IP the mon is pinned to (`nodeSelector`). This is because there is no `rook-ceph-mon-*` service created in that "mode". ### Restart the mon @@ -218,7 +210,8 @@ You will need to "restart" the good mon pod with the original `ceph-mon` command kubectl replace --force -f rook-ceph-mon-b-deployment.yaml ``` -> **NOTE**: Option `--force` will delete the deployment and create a new one +!!! note + Option `--force` will delete the deployment and create a new one Start the rook [toolbox](/Documentation/ceph-toolbox.md) and verify the status of the cluster. @@ -229,7 +222,9 @@ ceph -s The status should show one mon in quorum. If the status looks good, your cluster should be healthy again. ### Restart the operator + Start the rook operator again to resume monitoring the health of the cluster. + ```console # create the operator. it is safe to ignore the errors that a number of resources already exist. kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas=1 @@ -306,7 +301,7 @@ all ceph daemons will continue running and there will be no downtime. 5. Create the CephCluster CR with the same settings as previously -```shell +```console # Use the same cluster settings as exported above in step 2. kubectl create -f cluster.yaml ``` @@ -319,7 +314,7 @@ kubectl create -f cluster.yaml 7. Scale up the operator -```shell +```console kubectl -n rook-ceph scale --replicas=1 deploy/rook-ceph-operator ``` @@ -366,7 +361,7 @@ Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to 1. **STATE**: Now the cluster will have `rook-ceph-mon-a`, `rook-ceph-mgr-a`, and all the auxiliary pods up and running, and zero (hopefully) `rook-ceph-osd-ID-xxxxxx` running. `ceph -s` output should report 1 mon, 1 mgr running, and all of the OSDs down, all PGs are in `unknown` state. Rook should not start any OSD daemon since all devices belongs to the old cluster (which have a different `fsid`). 1. Run `kubectl -n rook-ceph exec -it rook-ceph-mon-a-xxxxxxxx bash` to enter the `rook-ceph-mon-a` pod, - ```shell + ```console mon-a# cat /etc/ceph/keyring-store/keyring # save this keyring content for later use mon-a# exit ``` @@ -381,7 +376,7 @@ Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to 3. Replace `/var/lib/rook/mon-a/keyring` with the saved keyring, preserving only the `[mon.]` section, remove `[client.admin]` section. 4. Run `docker run -it --rm -v /var/lib/rook:/var/lib/rook ceph/ceph:v14.2.1-20190430 bash`. The Docker image tag should match the Ceph version used in the Rook cluster. The `/etc/ceph/ceph.conf` file needs to exist for `ceph-mon` to work. - ```shell + ```console touch /etc/ceph/ceph.conf cd /var/lib/rook ceph-mon --extract-monmap monmap --mon-data ./mon-a/data # Extract monmap from old ceph-mon db and save as monmap @@ -399,8 +394,8 @@ Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to 1. Tell Rook to run as old cluster by running `kubectl -n rook-ceph edit secret/rook-ceph-mon` and changing `fsid` to the original `fsid`. Note that the `fsid` is base64 encoded and must not contain a trailing carriage return. For example: - ```shell - $ echo -n a811f99a-d865-46b7-8f2c-f94c064e4356 | base64 # Replace with the fsid from your old cluster. + ```console + echo -n a811f99a-d865-46b7-8f2c-f94c064e4356 | base64 # Replace with the fsid from your old cluster. ``` 1. Disable authentication by running `kubectl -n rook-ceph edit cm/rook-config-override` and adding content below: @@ -436,12 +431,14 @@ Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to ## Backing up and restoring a cluster based on PVCs into a new Kubernetes cluster -It is possible to migrate/restore an rook/ceph cluster from an existing Kubernetes cluster to a new one without resorting to SSH access or ceph tooling. This allows doing the migration using standard kubernetes resources only. This guide assumes the following +It is possible to migrate/restore an rook/ceph cluster from an existing Kubernetes cluster to a new one without resorting to SSH access or ceph tooling. This allows doing the migration using standard kubernetes resources only. This guide assumes the following: + 1. You have a CephCluster that uses PVCs to persist mon and osd data. Let's call it the "old cluster" -1. You can restore the PVCs as-is in the new cluster. Usually this is done by taking regular snapshots of the PVC volumes and using a tool that can re-create PVCs from these snapshots in the underlying cloud provider. Velero is one such tool. (https://github.com/vmware-tanzu/velero) +1. You can restore the PVCs as-is in the new cluster. Usually this is done by taking regular snapshots of the PVC volumes and using a tool that can re-create PVCs from these snapshots in the underlying cloud provider. [Velero](https://github.com/vmware-tanzu/velero) is one such tool. 1. You have regular backups of the secrets and configmaps in the rook-ceph namespace. Velero provides this functionality too. Do the following in the new cluster: + 1. Stop the rook operator by scaling the deployment `rook-ceph-operator` down to zero: `kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas 0` and deleting the other deployments. An example command to do this is `k -n rook-ceph delete deployment -l operator!=rook` 1. Restore the rook PVCs to the new cluster. diff --git a/Documentation/ceph-openshift-issues.md b/Documentation/Troubleshooting/openshift-common-issues.md similarity index 54% rename from Documentation/ceph-openshift-issues.md rename to Documentation/Troubleshooting/openshift-common-issues.md index a51316837116..cdb88b632a7e 100644 --- a/Documentation/ceph-openshift-issues.md +++ b/Documentation/Troubleshooting/openshift-common-issues.md @@ -1,11 +1,7 @@ --- title: OpenShift Common Issues -weight: 11400 -indent: true --- -# OpenShift Common Issues - ## Enable Monitoring in the Storage Dashboard OpenShift Console uses OpenShift Prometheus for monitoring and populating data in Storage Dashboard. Additional configuration is required to monitor the Ceph Cluster from the storage dashboard. @@ -14,7 +10,7 @@ OpenShift Console uses OpenShift Prometheus for monitoring and populating data i Change the namespace of the RoleBinding `rook-ceph-metrics` from `rook-ceph` to `openshift-monitoring` for the `prometheus-k8s` ServiceAccount in [rbac.yaml](https://github.com/rook/rook/blob/master/deploy/examples/monitoring/rbac.yaml#L70). -``` +```yaml subjects: - kind: ServiceAccount name: prometheus-k8s @@ -23,45 +19,39 @@ subjects: 2. Enable Ceph Cluster monitoring - Follow [ceph-monitoring/prometheus-alerts](ceph-monitoring.md#prometheus-alerts). + Follow [ceph-monitoring/prometheus-alerts](../Storage-Configuration/Monitoring/ceph-monitoring.md#prometheus-alerts). 3. Set the required label on the namespace - `oc label namespace rook-ceph "openshift.io/cluster-monitoring=true"` + ```console + oc label namespace rook-ceph "openshift.io/cluster-monitoring=true" + ``` ## Troubleshoot Monitoring Issues -> **Pre-req:** Switch to `rook-ceph` namespace with `oc project rook-ceph` +!!! attention + Switch to `rook-ceph` namespace using `oc project rook-ceph`. 1. Ensure ceph-mgr pod is Running ```console - oc get pods -l app=rook-ceph-mgr + $ oc get pods -l app=rook-ceph-mgr + NAME READY STATUS RESTARTS AGE + rook-ceph-mgr 1/1 Running 0 14h ``` - >``` - >NAME READY STATUS RESTARTS AGE - >rook-ceph-mgr 1/1 Running 0 14h - >``` - 2. Ensure service monitor is present ```console - oc get servicemonitor rook-ceph-mgr + $ oc get servicemonitor rook-ceph-mgr + NAME AGE + rook-ceph-mgr 14h ``` - >``` - >NAME AGE - >rook-ceph-mgr 14h - >``` - -3. Ensure prometheus rules are present +3. Ensure the prometheus rules object has been created ```console - oc get prometheusrules -l prometheus=rook-prometheus + $ oc get prometheusrules -l prometheus=rook-prometheus + NAME AGE + prometheus-ceph-rules 14h ``` - - >``` - >NAME AGE - >prometheus-ceph-rules 14h - >``` diff --git a/Documentation/Upgrade/.pages b/Documentation/Upgrade/.pages new file mode 100644 index 000000000000..578fbe579965 --- /dev/null +++ b/Documentation/Upgrade/.pages @@ -0,0 +1,3 @@ +nav: + - rook-upgrade.md + - ... diff --git a/Documentation/ceph-upgrade.md b/Documentation/Upgrade/rook-upgrade.md similarity index 88% rename from Documentation/ceph-upgrade.md rename to Documentation/Upgrade/rook-upgrade.md index e966c06850f7..a04a31470451 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/Upgrade/rook-upgrade.md @@ -1,13 +1,7 @@ --- -title: Upgrades -weight: 3800 -indent: true +title: Rook Upgrades --- -{% include_relative branch.liquid %} - -# Rook-Ceph Upgrades - This guide will walk you through the steps to upgrade the software in a Rook-Ceph cluster from one version to the next. This includes both the Rook-Ceph operator software itself as well as the Ceph cluster software. @@ -80,7 +74,8 @@ example, when Rook v1.9.1 is released, the process of updating from v1.9.0 is as the following: First get the latest common resources manifests that contain the latest changes for Rook v1.9. -```sh + +```console git clone --single-branch --depth=1 --branch v1.9.1 https://github.com/rook/rook.git cd rook/deploy/examples ``` @@ -90,6 +85,7 @@ If you have deployed the Rook Operator or the Ceph cluster into a different name section for instructions on how to change the default namespaces in `common.yaml`. Then apply the latest changes from v1.9 and update the Rook Operator image. + ```console kubectl apply -f common.yaml -f crds.yaml kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.9.1 @@ -124,8 +120,8 @@ time without compatibility support and without prior notice. We will do all our work in the Ceph example manifests directory. -```sh -$ cd $YOUR_ROOK_REPO/deploy/examples/ +```console +cd $YOUR_ROOK_REPO/deploy/examples/ ``` Unless your Rook cluster was created with customized namespaces, namespaces for Rook clusters are @@ -139,7 +135,7 @@ With this guide, we do our best not to assume the namespaces in your cluster. To as possible, modify and use the below snippet to configure your environment. We will use these environment variables throughout this document. -```sh +```console # Parameterize the environment export ROOK_OPERATOR_NAMESPACE="rook-ceph" export ROOK_CLUSTER_NAMESPACE="rook-ceph" @@ -159,19 +155,19 @@ Before we begin the upgrade process, let's first review some ways that you can v your cluster, ensuring that the upgrade is going smoothly after each step. Most of the health verification checks for your cluster during the upgrade process can be performed with the Rook toolbox. For more information about how to run the toolbox, please visit the -[Rook toolbox readme](./ceph-toolbox.md). +[Rook toolbox readme](../Troubleshooting/ceph-toolbox.md). See the common issues pages for troubleshooting and correcting health issues: -* [General troubleshooting](./common-issues.md) -* [Ceph troubleshooting](./ceph-common-issues.md) +* [General troubleshooting](../Troubleshooting/common-issues.md) +* [Ceph troubleshooting](../Troubleshooting/ceph-common-issues.md) ### **Pods all Running** In a healthy Rook cluster, the operator, the agents and all Rook namespace pods should be in the `Running` state and have few, if any, pod restarts. To verify this, run the following commands: -```sh +```console kubectl -n $ROOK_CLUSTER_NAMESPACE get pods ``` @@ -180,34 +176,31 @@ kubectl -n $ROOK_CLUSTER_NAMESPACE get pods The Rook toolbox contains the Ceph tools that can give you status details of the cluster with the `ceph status` command. Let's look at an output sample and review some of the details: -```sh -TOOLS_POD=$(kubectl -n $ROOK_CLUSTER_NAMESPACE get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[*].metadata.name}') -kubectl -n $ROOK_CLUSTER_NAMESPACE exec -it $TOOLS_POD -- ceph status +```console +$ TOOLS_POD=$(kubectl -n $ROOK_CLUSTER_NAMESPACE get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[*].metadata.name}') +$ kubectl -n $ROOK_CLUSTER_NAMESPACE exec -it $TOOLS_POD -- ceph status + cluster: + id: a3f4d647-9538-4aff-9fd1-b845873c3fe9 + health: HEALTH_OK + + services: + mon: 3 daemons, quorum b,c,a + mgr: a(active) + mds: myfs-1/1/1 up {0=myfs-a=up:active}, 1 up:standby-replay + osd: 6 osds: 6 up, 6 in + rgw: 1 daemon active + + data: + pools: 9 pools, 900 pgs + objects: 67 objects, 11 KiB + usage: 6.1 GiB used, 54 GiB / 60 GiB avail + pgs: 900 active+clean + + io: + client: 7.4 KiB/s rd, 681 B/s wr, 11 op/s rd, 4 op/s wr + recovery: 164 B/s, 1 objects/s ``` ->``` -> cluster: -> id: a3f4d647-9538-4aff-9fd1-b845873c3fe9 -> health: HEALTH_OK -> -> services: -> mon: 3 daemons, quorum b,c,a -> mgr: a(active) -> mds: myfs-1/1/1 up {0=myfs-a=up:active}, 1 up:standby-replay -> osd: 6 osds: 6 up, 6 in -> rgw: 1 daemon active -> -> data: -> pools: 9 pools, 900 pgs -> objects: 67 objects, 11 KiB -> usage: 6.1 GiB used, 54 GiB / 60 GiB avail -> pgs: 900 active+clean -> -> io: -> client: 7.4 KiB/s rd, 681 B/s wr, 11 op/s rd, 4 op/s wr -> recovery: 164 B/s, 1 objects/s ->``` - In the output above, note the following indications that the cluster is in a healthy state: * Cluster health: The overall cluster status is `HEALTH_OK` and there are no warning or error status @@ -227,7 +220,7 @@ details on the health of the system, such as `ceph osd status`. See the Rook will prevent the upgrade of the Ceph daemons if the health is in a `HEALTH_ERR` state. If you desired to proceed with the upgrade anyway, you will need to set either `skipUpgradeChecks: true` or `continueUpgradeAfterChecksEvenIfNotHealthy: true` as described in the -[cluster CR settings](ceph-cluster-crd.md#cluster-settings). +[cluster CR settings](../CRDs/ceph-cluster-crd.md#cluster-settings). ### **Container Versions** @@ -235,7 +228,7 @@ The container version running in a specific pod in the Rook cluster can be verif output. For example for the monitor pod `mon-b`, we can verify the container version it is running with the below commands: -```sh +```console POD_NAME=$(kubectl -n $ROOK_CLUSTER_NAMESPACE get pod -o custom-columns=name:.metadata.name --no-headers | grep rook-ceph-mon-b) kubectl -n $ROOK_CLUSTER_NAMESPACE get pod ${POD_NAME} -o jsonpath='{.spec.containers[0].image}' ``` @@ -243,7 +236,7 @@ kubectl -n $ROOK_CLUSTER_NAMESPACE get pod ${POD_NAME} -o jsonpath='{.spec.conta The status and container versions for all Rook pods can be collected all at once with the following commands: -```sh +```console kubectl -n $ROOK_OPERATOR_NAMESPACE get pod -o jsonpath='{range .items[*]}{.metadata.name}{"\n\t"}{.status.phase}{"\t\t"}{.spec.containers[0].image}{"\t"}{.spec.initContainers[0]}{"\n"}{end}' && \ kubectl -n $ROOK_CLUSTER_NAMESPACE get pod -o jsonpath='{range .items[*]}{.metadata.name}{"\n\t"}{.status.phase}{"\t\t"}{.spec.containers[0].image}{"\t"}{.spec.initContainers[0].image}{"\n"}{end}' ``` @@ -254,7 +247,7 @@ requested, updated, and currently available replicas for various Rook-Ceph resou the version of Rook for resources managed by the updated Rook-Ceph operator. Note that the operator and toolbox deployments do not have a `rook-version` label set. -```sh +```console kubectl -n $ROOK_CLUSTER_NAMESPACE get deployments -o jsonpath='{range .items[*]}{.metadata.name}{" \treq/upd/avl: "}{.spec.replicas}{"/"}{.status.updatedReplicas}{"/"}{.status.readyReplicas}{" \trook-version="}{.metadata.labels.rook-version}{"\n"}{end}' kubectl -n $ROOK_CLUSTER_NAMESPACE get jobs -o jsonpath='{range .items[*]}{.metadata.name}{" \tsucceeded: "}{.status.succeeded}{" \trook-version="}{.metadata.labels.rook-version}{"\n"}{end}' @@ -289,13 +282,15 @@ Let's get started! ### **1. Update common resources and CRDs** -> Automatically updated if you are upgrading via the helm chart +!! hint + If you are upgrading via the Helm chart, the common resources and CRDs are automatically updated. First apply updates to Rook-Ceph common resources. This includes modified privileges (RBAC) needed by the Operator. Also update the Custom Resource Definitions (CRDs). Get the latest common resources manifests that contain the latest changes. -```sh + +```console git clone --single-branch --depth=1 --branch v1.9.0 https://github.com/rook/rook.git cd rook/deploy/examples ``` @@ -303,7 +298,8 @@ cd rook/deploy/examples If you have deployed the Rook Operator or the Ceph cluster into a different namespace than `rook-ceph`, update the common resource manifests to use your `ROOK_OPERATOR_NAMESPACE` and `ROOK_CLUSTER_NAMESPACE` using `sed`. -```sh + +```console sed -i.bak \ -e "s/\(.*\):.*# namespace:operator/\1: $ROOK_OPERATOR_NAMESPACE # namespace:operator/g" \ -e "s/\(.*\):.*# namespace:cluster/\1: $ROOK_CLUSTER_NAMESPACE # namespace:cluster/g" \ @@ -311,7 +307,8 @@ sed -i.bak \ ``` Then apply the latest changes. -```sh + +```console kubectl apply -f common.yaml -f crds.yaml ``` @@ -319,10 +316,10 @@ kubectl apply -f common.yaml -f crds.yaml #### **Prometheus** -If you have [Prometheus monitoring](ceph-monitoring.md) enabled, follow the +If you have [Prometheus monitoring](../Storage-Configuration/Monitoring/ceph-monitoring.md) enabled, follow the step to upgrade the Prometheus RBAC resources as well. -```sh +```console kubectl apply -f deploy/examples/monitoring/rbac.yaml ``` @@ -333,32 +330,35 @@ additionally use `monitoring.createPrometheusRules` to instruct Helm to deploy t alternately deploy the rules manually if you wish. To see the latest information about manually deploying rules, see the -[Prometheus monitoring docs](ceph-monitoring.md#prometheus-alets). +[Prometheus monitoring docs](../Storage-Configuration/Monitoring/ceph-monitoring.md#prometheus-alets). #### **MDS liveness and startup probes** If you configure MDS probes in the CephCluster resource, copy them to the -[CephFilesystem `metadataServer` settings](ceph-filesystem-crd.md#metadata-server-settings) at this +[CephFilesystem `metadataServer` settings](../CRDs/Shared-Filesystem/ceph-filesystem-crd.md#metadata-server-settings) at this point. Do not remove them from the CephCluster until after the Rook upgrade is fully complete. ### **2. Update Ceph CSI versions** -> Automatically updated if you are upgrading via the helm chart +!! hint + If you are upgrading via the Helm chart and have not customized the CSI image versions, this is automatically updated. If you have specified custom CSI images in the Rook-Ceph Operator deployment, we recommended you update to use the latest Ceph-CSI drivers. See the [CSI Version](#csi-version) section for more details. -> Note: If using snapshots, refer to the [Upgrade Snapshot API guide](ceph-csi-snapshot.md#upgrade-snapshot-api). +!!! note + If using snapshots, refer to the [Upgrade Snapshot API guide](../Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md#upgrade-snapshot-api). ### **3. Update the Rook Operator** -> Automatically updated if you are upgrading via the helm chart +!! hint + If you are upgrading via the Helm chart, the operator is automatically updated. The largest portion of the upgrade is triggered when the operator's image is updated to `v1.9.x`. When the operator is updated, it will proceed to update all of the Ceph daemons. -```sh +```console kubectl -n $ROOK_OPERATOR_NAMESPACE set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.9.0 ``` @@ -370,24 +370,24 @@ and the Ceph Filesystem may fall offline a few times while the MDSes are upgradi The versions of the components can be viewed as they are updated: -```sh +```console watch --exec kubectl -n $ROOK_CLUSTER_NAMESPACE get deployments -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{.metadata.name}{" \treq/upd/avl: "}{.spec.replicas}{"/"}{.status.updatedReplicas}{"/"}{.status.readyReplicas}{" \trook-version="}{.metadata.labels.rook-version}{"\n"}{end}' ``` As an example, this cluster is midway through updating the OSDs. When all deployments report `1/1/1` availability and `rook-version=v1.9.0`, the Ceph cluster's core components are fully updated. ->``` ->Every 2.0s: kubectl -n rook-ceph get deployment -o j... -> ->rook-ceph-mgr-a req/upd/avl: 1/1/1 rook-version=v1.9.0 ->rook-ceph-mon-a req/upd/avl: 1/1/1 rook-version=v1.9.0 ->rook-ceph-mon-b req/upd/avl: 1/1/1 rook-version=v1.9.0 ->rook-ceph-mon-c req/upd/avl: 1/1/1 rook-version=v1.9.0 ->rook-ceph-osd-0 req/upd/avl: 1// rook-version=v1.9.0 ->rook-ceph-osd-1 req/upd/avl: 1/1/1 rook-version=v1.8.8 ->rook-ceph-osd-2 req/upd/avl: 1/1/1 rook-version=v1.8.8 ->``` +```console +Every 2.0s: kubectl -n rook-ceph get deployment -o j... + +rook-ceph-mgr-a req/upd/avl: 1/1/1 rook-version=v1.9.0 +rook-ceph-mon-a req/upd/avl: 1/1/1 rook-version=v1.9.0 +rook-ceph-mon-b req/upd/avl: 1/1/1 rook-version=v1.9.0 +rook-ceph-mon-c req/upd/avl: 1/1/1 rook-version=v1.9.0 +rook-ceph-osd-0 req/upd/avl: 1// rook-version=v1.9.0 +rook-ceph-osd-1 req/upd/avl: 1/1/1 rook-version=v1.8.8 +rook-ceph-osd-2 req/upd/avl: 1/1/1 rook-version=v1.8.8 +``` An easy check to see if the upgrade is totally finished is to check that there is only one `rook-version` reported across the cluster. @@ -407,18 +407,19 @@ At this point, your Rook operator should be running version `rook/ceph:v1.9.0`. Verify the Ceph cluster's health using the [health verification section](#health-verification). - ## Ceph Version Upgrades Rook v1.9 supports the following Ceph versions: -- Ceph Quincy v17.2.0 or newer -- Ceph Pacific v16.2.0 or newer -- Ceph Octopus v15.2.0 or newer + +* Ceph Quincy v17.2.0 or newer +* Ceph Pacific v16.2.0 or newer +* Ceph Octopus v15.2.0 or newer These are the only supported versions of Ceph. Rook v1.10 is planning to drop support for Ceph Octopus (15.2.x), so please consider upgrading your Ceph cluster. -> **IMPORTANT: When an update is requested, the operator will check Ceph's status, if it is in `HEALTH_ERR` it will refuse to do the upgrade.** +!!! important + When an update is requested, the operator will check Ceph's status, **if it is in `HEALTH_ERR` the operator will refuse to do the upgrade.** Rook is cautious when performing upgrades. When an upgrade is requested (the Ceph image has been updated in the CR), Rook will go through all the daemons one by one and will individually perform @@ -432,19 +433,22 @@ We recommend updating to v16.2.7 or newer. If you require updating **to v16.2.0- please see the [v1.8 upgrade guide for a special upgrade consideration](https://rook.github.io/docs/rook/v1.8/ceph-upgrade.html#disable-bluestore_fsck_quick_fix_on_mount). ### **Rename CephBlockPool device_health_metrics pool when upgrading to Quincy v17** + In Ceph Quincy (v17), the `device_health_metrics` pool was renamed to `.mgr`. Ceph will perform this migration automatically. If you do not use CephBlockPool to customize the configuration of the `device_health_metrics` pool, you don't need to do anything further here. If you do use CephBlockPool to customize the configuration of the `device_health_metrics` pool, you will need two extra steps after the Ceph upgrade is complete. Once upgrade is complete: + 1. Create a new CephBlockPool to configure the `.mgr` built-in pool. You can reference the example -[builtin mgr pool](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/pool-builtin-mgr.yaml). +[builtin mgr pool](https://github.com/rook/rook/blob/master/deploy/examples/pool-builtin-mgr.yaml). 2. Delete the old CephBlockPool that represents the `device_health_metrics` pool. ### **Important consideration for CephNFS users** + Ceph Quincy v17.2.0 has a potentially breaking regression with CephNFS. See the NFS documentation's -[known issue](ceph-nfs-crd.md#ceph-v1720) for more detail. +[known issue](../CRDs/ceph-nfs-crd.md#ceph-v1720) for more detail. ### **Ceph images** @@ -468,7 +472,7 @@ These images are tagged in a few ways: The majority of the upgrade will be handled by the Rook operator. Begin the upgrade by changing the Ceph image field in the cluster CRD (`spec.cephVersion.image`). -```sh +```console NEW_CEPH_IMAGE='quay.io/ceph/ceph:v16.2.7-20220216' CLUSTER_NAME="$ROOK_CLUSTER_NAMESPACE" # change if your cluster name is not the Rook namespace kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $CLUSTER_NAME --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}" @@ -479,7 +483,7 @@ kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $CLUSTER_NAME --type=merge As with upgrading Rook, you must now wait for the upgrade to complete. Status can be determined in a similar way to the Rook upgrade as well. -```sh +```console watch --exec kubectl -n $ROOK_CLUSTER_NAMESPACE get deployments -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{.metadata.name}{" \treq/upd/avl: "}{.spec.replicas}{"/"}{.status.updatedReplicas}{"/"}{.status.readyReplicas}{" \tceph-version="}{.metadata.labels.ceph-version}{"\n"}{end}' ``` @@ -498,7 +502,6 @@ This cluster is finished: Verify the Ceph cluster's health using the [health verification section](#health-verification). - ## CSI Version If you have a cluster running with CSI drivers enabled and you want to configure Rook @@ -541,4 +544,4 @@ which CSI features are enabled. kubectl --namespace rook-ceph get pod -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{"\n"}' -l 'app in (csi-rbdplugin,csi-rbdplugin-provisioner,csi-cephfsplugin,csi-cephfsplugin-provisioner)' | sort | uniq ``` -The default images can also be found with each release in the [images list](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/images.txt) +The default images can also be found with each release in the [images list](https://github.com/rook/rook/blob/master/deploy/examples/images.txt) diff --git a/Documentation/branch.liquid b/Documentation/branch.liquid deleted file mode 100644 index 0ce69039e9db..000000000000 --- a/Documentation/branch.liquid +++ /dev/null @@ -1,7 +0,0 @@ -{% assign url = page.url | split: '/' %} -{% assign currentVersion = url[3] %} -{% if currentVersion != 'latest' %} -{% assign branchName = currentVersion | replace: 'v', '' | prepend: 'release-' %} -{% else %} -{% assign branchName = 'master' %} -{% endif %} diff --git a/Documentation/ceph-examples.md b/Documentation/ceph-examples.md deleted file mode 100644 index ad30ccf8412e..000000000000 --- a/Documentation/ceph-examples.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -title: Examples -weight: 2050 -indent: true ---- -{% include_relative branch.liquid %} - -# Ceph Examples - -Configuration for Rook and Ceph can be configured in multiple ways to provide block devices, shared filesystem volumes or object storage in a kubernetes namespace. We have provided several examples to simplify storage setup, but remember there are many tunables and you will need to decide what settings work for your use case and environment. - -See the **[example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples)** folder for all the rook/ceph setup example spec files. - -## Common Resources - -The first step to deploy Rook is to create the CRDs and other common resources. The configuration for these resources will be the same for most deployments. -The [crds.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/crds.yaml) and -[common.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/common.yaml) sets these resources up. - -```console -kubectl create -f crds.yaml -f common.yaml -``` - -The examples all assume the operator and all Ceph daemons will be started in the same namespace. If you want to deploy the operator in a separate namespace, see the comments throughout `common.yaml`. - -## Operator - -After the common resources are created, the next step is to create the Operator deployment. Several spec file examples are provided in [this directory](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/): - -* `operator.yaml`: The most common settings for production deployments - * `kubectl create -f operator.yaml` -* `operator-openshift.yaml`: Includes all of the operator settings for running a basic Rook cluster in an OpenShift environment. You will also want to review the [OpenShift Prerequisites](ceph-openshift.md) to confirm the settings. - * `oc create -f operator-openshift.yaml` - -Settings for the operator are configured through environment variables on the operator deployment. The individual settings are documented in [operator.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/operator.yaml). - -## Cluster CRD - -Now that your operator is running, let's create your Ceph storage cluster. This CR contains the most critical settings -that will influence how the operator configures the storage. It is important to understand the various ways to configure -the cluster. These examples represent a very small set of the different ways to configure the storage. - -* `cluster.yaml`: This file contains common settings for a production storage cluster. Requires at least three worker nodes. -* `cluster-test.yaml`: Settings for a test cluster where redundancy is not configured. Requires only a single node. -* `cluster-on-pvc.yaml`: This file contains common settings for backing the Ceph Mons and OSDs by PVs. Useful when running in cloud environments or where local PVs have been created for Ceph to consume. -* `cluster-external.yaml`: Connect to an [external Ceph cluster](ceph-cluster-crd.md#external-cluster) with minimal access to monitor the health of the cluster and connect to the storage. -* `cluster-external-management.yaml`: Connect to an [external Ceph cluster](ceph-cluster-crd.md#external-cluster) with the admin key of the external cluster to enable - remote creation of pools and configure services such as an [Object Store](ceph-object.md) or a [Shared Filesystem](ceph-filesystem.md). -* `cluster-stretched.yaml`: Create a cluster in "stretched" mode, with five mons stretched across three zones, and the OSDs across two zones. See the [Stretch documentation](ceph-cluster-crd.md#stretch-cluster). - -See the [Cluster CRD](ceph-cluster-crd.md) topic for more details and more examples for the settings. - -## Setting up consumable storage - -Now we are ready to setup [block](https://ceph.com/ceph-storage/block-storage/), [shared filesystem](https://ceph.com/ceph-storage/file-system/) or [object storage](https://ceph.com/ceph-storage/object-storage/) in the Rook Ceph cluster. These kinds of storage are respectively referred to as CephBlockPool, CephFilesystem and CephObjectStore in the spec files. - -### Block Devices - -Ceph can provide raw block device volumes to pods. Each example below sets up a storage class which can then be used to provision a block device in kubernetes pods. The storage class is defined with [a pool](http://docs.ceph.com/docs/master/rados/operations/pools/) which defines the level of data redundancy in Ceph: - -* `storageclass.yaml`: This example illustrates replication of 3 for production scenarios and requires at least three worker nodes. Your data is replicated on three different kubernetes worker nodes and intermittent or long-lasting single node failures will not result in data unavailability or loss. -* `storageclass-ec.yaml`: Configures erasure coding for data durability rather than replication. [Ceph's erasure coding](http://docs.ceph.com/docs/master/rados/operations/erasure-code/) is more efficient than replication so you can get high reliability without the 3x replication cost of the preceding example (but at the cost of higher computational encoding and decoding costs on the worker nodes). Erasure coding requires at least three worker nodes. See the [Erasure coding](ceph-pool-crd.md#erasure-coded) documentation for more details. -* `storageclass-test.yaml`: Replication of 1 for test scenarios and it requires only a single node. Do not use this for applications that store valuable data or have high-availability storage requirements, since a single node failure can result in data loss. - -The storage classes are found in different sub-directories depending on the driver: - -* `csi/rbd`: The CSI driver for block devices. This is the preferred driver going forward. - -See the [Ceph Pool CRD](ceph-pool-crd.md) topic for more details on the settings. - -### Shared Filesystem - -Ceph filesystem (CephFS) allows the user to 'mount' a shared posix-compliant folder into one or more hosts (pods in the container world). This storage is similar to NFS shared storage or CIFS shared folders, as explained [here](https://ceph.com/ceph-storage/file-system/). - -File storage contains multiple pools that can be configured for different scenarios: - -* `filesystem.yaml`: Replication of 3 for production scenarios. Requires at least three worker nodes. -* `filesystem-ec.yaml`: Erasure coding for production scenarios. Requires at least three worker nodes. -* `filesystem-test.yaml`: Replication of 1 for test scenarios. Requires only a single node. - -Dynamic provisioning is possible with the CSI driver. The storage class for shared filesystems is found in the `csi/cephfs` directory. - -See the [Shared Filesystem CRD](ceph-filesystem-crd.md) topic for more details on the settings. - -### Object Storage - -Ceph supports storing blobs of data called objects that support HTTP(s)-type get/put/post and delete semantics. This storage is similar to AWS S3 storage, for example. - -Object storage contains multiple pools that can be configured for different scenarios: - -* `object.yaml`: Replication of 3 for production scenarios. Requires at least three worker nodes. -* `object-openshift.yaml`: Replication of 3 with rgw in a port range valid for OpenShift. Requires at least three worker nodes. -* `object-ec.yaml`: Erasure coding rather than replication for production scenarios. Requires at least three worker nodes. -* `object-test.yaml`: Replication of 1 for test scenarios. Requires only a single node. - -See the [Object Store CRD](ceph-object-store-crd.md) topic for more details on the settings. - -### Object Storage User - -- `object-user.yaml`: Creates a simple object storage user and generates credentials for the S3 API - -### Object Storage Buckets - -The Ceph operator also runs an object store bucket provisioner which can grant access to existing buckets or dynamically provision new buckets. - -* [object-bucket-claim-retain.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/object-bucket-claim-retain.yaml) Creates a request for a new bucket by referencing a StorageClass which saves the bucket when the initiating OBC is deleted. -* [object-bucket-claim-delete.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/object-bucket-claim-delete.yaml) Creates a request for a new bucket by referencing a StorageClass which deletes the bucket when the initiating OBC is deleted. -* [storageclass-bucket-retain.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/storageclass-bucket-retain.yaml) Creates a new StorageClass which defines the Ceph Object Store and retains the bucket after the initiating OBC is deleted. -* [storageclass-bucket-delete.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/storageclass-bucket-delete.yaml) Creates a new StorageClass which defines the Ceph Object Store and deletes the bucket after the initiating OBC is deleted. diff --git a/Documentation/ceph-tools.md b/Documentation/ceph-tools.md deleted file mode 100644 index a715d76f0b58..000000000000 --- a/Documentation/ceph-tools.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: Ceph Tools -weight: 11000 ---- - -# Ceph Tools - -Rook provides a number of tools and troubleshooting docs to help you manage your cluster. - -* [Toolbox](ceph-toolbox.md): A pod from which you can run all of the tools to troubleshoot the storage cluster -* [Common Issues](ceph-common-issues.md): Common issues and their potential solutions -* [OSD Management](ceph-osd-mgmt.md): Common configuration issues for Ceph OSDs such as adding and removing storage -* [Direct Tools](direct-tools.md): Run ceph commands to test directly mounting block and file storage -* [Advanced Configuration](ceph-advanced-configuration.md): Tips and tricks for configuring for cluster -* [Openshift Common Issues](ceph-openshift-issues.md): Common troubleshooting tips for OpenShift clusters -* [Disaster Recovery](ceph-disaster-recovery.md): In the worst case scenario if the ceph mons lose quorum, follow these steps to recover diff --git a/Documentation/helm.md b/Documentation/helm.md deleted file mode 100644 index f71f8e3a2052..000000000000 --- a/Documentation/helm.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Helm Charts -weight: 10000 ---- - -{% include_relative branch.liquid %} - -# Helm Charts - -Rook has published the following Helm charts for the Ceph storage provider: - -* [Rook Ceph Operator](helm-operator.md): Starts the Ceph Operator, which will watch for Ceph CRs (custom resources) -* [Rook Ceph Cluster](helm-ceph-cluster.md): Creates Ceph CRs that the operator will use to configure the cluster - -The Helm charts are intended to simplify deployment and upgrades. -Configuring the Rook resources without Helm is also fully supported by creating the -[manifests](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples) -directly. diff --git a/Documentation/quickstart.md b/Documentation/quickstart.md deleted file mode 100644 index f76bb7c2ae98..000000000000 --- a/Documentation/quickstart.md +++ /dev/null @@ -1,172 +0,0 @@ ---- -title: Quickstart -weight: 300 ---- - -{% include_relative branch.liquid %} - -# Ceph Quickstart - -Welcome to Rook! We hope you have a great experience installing the Rook **cloud-native storage orchestrator** platform to enable highly available, durable Ceph storage in your Kubernetes cluster. - -If you have any questions along the way, please don't hesitate to ask us in our [Slack channel](https://rook-io.slack.com). You can sign up for our Slack [here](https://slack.rook.io). - -This guide will walk you through the basic setup of a Ceph cluster and enable you to consume block, object, and file storage -from other pods running in your cluster. - -**Always use a virtual machine when testing Rook. Never use your host system where local devices may mistakenly be consumed.** - -## Minimum Version - -Kubernetes **v1.17** or higher is supported by Rook. - -## Prerequisites - -To make sure you have a Kubernetes cluster that is ready for `Rook`, you can [follow these instructions](pre-reqs.md). - -In order to configure the Ceph storage cluster, at least one of these local storage options are required: -- Raw devices (no partitions or formatted filesystems) - - This requires `lvm2` to be installed on the host. - To avoid this dependency, you can create a single full-disk partition on the disk (see below) -- Raw partitions (no formatted filesystem) -- Persistent Volumes available from a storage class in `block` mode - -## TL;DR - -A simple Rook cluster can be created with the following kubectl commands and [example manifests](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples). - -```console -$ git clone --single-branch --branch {{ branchName }} https://github.com/rook/rook.git -cd rook/deploy/examples -kubectl create -f crds.yaml -f common.yaml -f operator.yaml -kubectl create -f cluster.yaml -``` - -After the cluster is running, you can create [block, object, or file](#storage) storage to be consumed by other applications in your cluster. - -## Deploy the Rook Operator - -The first step is to deploy the Rook operator. Check that you are using the [example yaml files](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples) that correspond to your release of Rook. For more options, see the [examples documentation](ceph-examples.md). - -```console -cd deploy/examples -kubectl create -f crds.yaml -f common.yaml -f operator.yaml - -# verify the rook-ceph-operator is in the `Running` state before proceeding -kubectl -n rook-ceph get pod -``` - -You can also deploy the operator with the [Rook Helm Chart](helm-operator.md). - -Before you start the operator in production, there are some settings that you may want to consider: -1. Consider if you want to enable certain Rook features that are disabled by default. See the [operator.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/operator.yaml) for these and other advanced settings. - 1. Device discovery: Rook will watch for new devices to configure if the `ROOK_ENABLE_DISCOVERY_DAEMON` setting is enabled, commonly used in bare metal clusters. - 2. Node affinity and tolerations: The CSI driver by default will run on any node in the cluster. To configure the CSI driver affinity, several settings are available. - -If you wish to deploy into a namespace other than the default `rook-ceph`, see the -[Ceph advanced configuration section](ceph-advanced-configuration.md#using-alternate-namespaces) on the topic. - -## Cluster Environments - -The Rook documentation is focused around starting Rook in a production environment. Examples are also -provided to relax some settings for test environments. When creating the cluster later in this guide, consider these example cluster manifests: - -- [cluster.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml): Cluster settings for a production cluster running on bare metal. Requires at least three worker nodes. -- [cluster-on-pvc.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster-on-pvc.yaml): Cluster settings for a production cluster running in a dynamic cloud environment. -- [cluster-test.yaml](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster-test.yaml): Cluster settings for a test environment such as minikube. - -See the [Ceph examples](ceph-examples.md) for more details. - -## Create a Ceph Cluster - -Now that the Rook operator is running we can create the Ceph cluster. For the cluster to survive reboots, -make sure you set the `dataDirHostPath` property that is valid for your hosts. For more settings, see the documentation on [configuring the cluster](ceph-cluster-crd.md). - -Create the cluster: - -```console -kubectl create -f cluster.yaml -``` - -Use `kubectl` to list pods in the `rook-ceph` namespace. You should be able to see the following pods once they are all running. -The number of osd pods will depend on the number of nodes in the cluster and the number of devices configured. -If you did not modify the `cluster.yaml` above, it is expected that one OSD will be created per node. - -> If the `rook-ceph-mon`, `rook-ceph-mgr`, or `rook-ceph-osd` pods are not created, please refer to the -> [Ceph common issues](ceph-common-issues.md) for more details and potential solutions. - -```console -kubectl -n rook-ceph get pod -``` - ->``` ->NAME READY STATUS RESTARTS AGE ->csi-cephfsplugin-provisioner-d77bb49c6-n5tgs 5/5 Running 0 140s ->csi-cephfsplugin-provisioner-d77bb49c6-v9rvn 5/5 Running 0 140s ->csi-cephfsplugin-rthrp 3/3 Running 0 140s ->csi-rbdplugin-hbsm7 3/3 Running 0 140s ->csi-rbdplugin-provisioner-5b5cd64fd-nvk6c 6/6 Running 0 140s ->csi-rbdplugin-provisioner-5b5cd64fd-q7bxl 6/6 Running 0 140s ->rook-ceph-crashcollector-minikube-5b57b7c5d4-hfldl 1/1 Running 0 105s ->rook-ceph-mgr-a-64cd7cdf54-j8b5p 1/1 Running 0 77s ->rook-ceph-mon-a-694bb7987d-fp9w7 1/1 Running 0 105s ->rook-ceph-mon-b-856fdd5cb9-5h2qk 1/1 Running 0 94s ->rook-ceph-mon-c-57545897fc-j576h 1/1 Running 0 85s ->rook-ceph-operator-85f5b946bd-s8grz 1/1 Running 0 92m ->rook-ceph-osd-0-6bb747b6c5-lnvb6 1/1 Running 0 23s ->rook-ceph-osd-1-7f67f9646d-44p7v 1/1 Running 0 24s ->rook-ceph-osd-2-6cd4b776ff-v4d68 1/1 Running 0 25s ->rook-ceph-osd-prepare-node1-vx2rz 0/2 Completed 0 60s ->rook-ceph-osd-prepare-node2-ab3fd 0/2 Completed 0 60s ->rook-ceph-osd-prepare-node3-w4xyz 0/2 Completed 0 60s ->``` - -To verify that the cluster is in a healthy state, connect to the [Rook toolbox](ceph-toolbox.md) and run the -`ceph status` command. - -* All mons should be in quorum -* A mgr should be active -* At least one OSD should be active -* If the health is not `HEALTH_OK`, the warnings or errors should be investigated - -```console -ceph status -``` ->``` -> cluster: -> id: a0452c76-30d9-4c1a-a948-5d8405f19a7c -> health: HEALTH_OK -> -> services: -> mon: 3 daemons, quorum a,b,c (age 3m) -> mgr: a(active, since 2m) -> osd: 3 osds: 3 up (since 1m), 3 in (since 1m) ->... ->``` - -If the cluster is not healthy, please refer to the [Ceph common issues](ceph-common-issues.md) for more details and potential solutions. - -## Storage - -For a walkthrough of the three types of storage exposed by Rook, see the guides for: - -* **[Block](ceph-block.md)**: Create block storage to be consumed by a pod (RWO) -* **[Shared Filesystem](ceph-filesystem.md)**: Create a filesystem to be shared across multiple pods (RWX) -* **[Object](ceph-object.md)**: Create an object store that is accessible inside or outside the Kubernetes cluster - -## Ceph Dashboard - -Ceph has a dashboard in which you can view the status of your cluster. Please see the [dashboard guide](ceph-dashboard.md) for more details. - -## Tools - -Create a toolbox pod for full access to a ceph admin client for debugging and troubleshooting your Rook cluster. Please see the [toolbox documentation](ceph-toolbox.md) for setup and usage information. Also see our [advanced configuration](ceph-advanced-configuration.md) document for helpful maintenance and tuning examples. - -## Monitoring - -Each Rook cluster has some built in metrics collectors/exporters for monitoring with [Prometheus](https://prometheus.io/). -To learn how to set up monitoring for your Rook cluster, you can follow the steps in the [monitoring guide](./ceph-monitoring.md). - -## Teardown - -When you are done with the test cluster, see [these instructions](ceph-teardown.md) to clean up the cluster. diff --git a/Documentation/stylesheets/extra.css b/Documentation/stylesheets/extra.css new file mode 100644 index 000000000000..4d474129af44 --- /dev/null +++ b/Documentation/stylesheets/extra.css @@ -0,0 +1,3 @@ +[data-md-color-primary="rook-blue"] { + --md-primary-fg-color: #2ac6ea; +} diff --git a/build/release/Makefile b/build/release/Makefile index d34674b6dedc..a899248b41cb 100644 --- a/build/release/Makefile +++ b/build/release/Makefile @@ -31,19 +31,28 @@ ifneq ($(TAG_WITH_SUFFIX),true) override VERSION := $(shell echo "$(VERSION)" | sed -e 's/-alpha.0//' -e 's/-beta.0//' -e 's/-rc.0//') endif +DOCS_PREFIX := docs/rook + ifeq ($(shell echo $(BRANCH_NAME)),master) DOCS_VERSION := latest else DOCS_VERSION := $(shell echo $(BRANCH_NAME) | sed -E "s/^release\-([0-9]+)\.([0-9]+)$$/v\1.\2/g") endif + +ifeq ($(shell echo $(BRANCH_NAME)),$(shell git branch -r | grep 'origin/release-' | sort | tail -n 1 | cut -d'/' -f2)) +DOCS_VERSION_ALIAS := latest-release +else +DOCS_VERSION_ALIAS := +endif + DOCS_DIR ?= $(ROOT_DIR)/Documentation DOCS_WORK_DIR := $(WORK_DIR)/rook.github.io DOCS_VERSION_DIR := $(DOCS_WORK_DIR)/docs/rook/$(DOCS_VERSION) ifdef GIT_API_TOKEN -DOCS_GIT_REPO := https://$(GIT_API_TOKEN)@github.com/rook/rook.github.io.git +DOCS_GIT_REPO := https://$(GIT_API_TOKEN)@github.com/koor-tech/rook.github.io.git else -DOCS_GIT_REPO := git@github.com:rook/rook.github.io.git +DOCS_GIT_REPO := git@github.com:koor-tech/rook.github.io.git endif ifeq ($(origin BRANCH_NAME), undefined) @@ -122,17 +131,13 @@ endif # ==================================================================================== # docs - build.docs: - rm -rf $(DOCS_WORK_DIR) - mkdir -p $(DOCS_WORK_DIR) - git clone --depth=1 -b master $(DOCS_GIT_REPO) $(DOCS_WORK_DIR) - rm -rf $(DOCS_VERSION_DIR) - cp -r $(DOCS_DIR)/ $(DOCS_VERSION_DIR) - cd $(DOCS_WORK_DIR) && DOCS_VERSION=$(DOCS_VERSION) $(MAKE) _data/projects.json + pip install -r requirements_docs.txt publish.docs: - cd $(DOCS_WORK_DIR) && DOCS_VERSION=$(DOCS_VERSION) $(MAKE) publish + if ! git remote show docs > /dev/null 2>&1; then git remote add docs $(DOCS_GIT_REPO); fi + git fetch -u docs gh-pages + mike deploy --remote docs --push --rebase --branch gh-pages --update-aliases --prefix $(DOCS_PREFIX) $(DOCS_VERSION) $(DOCS_VERSION_ALIAS) # ==================================================================================== # helm diff --git a/design/ceph/decouple-ceph-version.md b/design/ceph/decouple-ceph-version.md index 38cdc9a32841..f90239b64b74 100644 --- a/design/ceph/decouple-ceph-version.md +++ b/design/ceph/decouple-ceph-version.md @@ -109,7 +109,7 @@ Daemons will only be restarted when necessary for the upgrade. The Rook upgrade depending on if the pod spec changed. The Ceph upgrade will always require a restart of the daemons. In either case, a restart will be done in an orderly, rolling manner with one pod at a time along with health checks as the upgrade proceeds. The upgrade will be paused if the cluster becomes unhealthy. -See the [Upgrade design doc](ceph-upgrade.md) for more details on the general upgrade approach. +See the [Upgrade design doc](rook-upgrade.md) for more details on the general upgrade approach. #### Admin control of upgrades diff --git a/design/ceph/upgrade.md b/design/ceph/upgrade.md index c26b32af207c..961a8068ed18 100644 --- a/design/ceph/upgrade.md +++ b/design/ceph/upgrade.md @@ -19,7 +19,7 @@ The cluster should be maintained in a healthy state the entire time. ## **User Guide** Until automated upgrade support is available in Rook, we have authored a user guide that walks you through the steps to upgrade the software in a Rook cluster. Consideration is also provided in the guide for how to verify the cluster remains healthy during and after the upgrade process. -Please refer to the [Rook Upgrade User Guide](../Documentation/ceph-upgrade.md) to learn more about the current Rook upgrade process. +Please refer to the [Rook Upgrade User Guide](../Documentation/rook-upgrade.md) to learn more about the current Rook upgrade process. ## **Detailed Design** The responsibility for performing and orchestrating an upgrade will be handled by an upgrade controller that runs as part of the Rook operator, in the same pod and process (similar to how the Rook volume provisioner is run). @@ -34,7 +34,7 @@ The upgrade controller should not begin an upgrade if the cluster is currently u ### **General Sequence** This section describes in a broad sense the general sequence of steps for upgrading a Rook cluster after a new Rook software version is released, e.g. `v0.6.1`. -Note that this sequence is modeled after the [Rook Upgrade User Guide](../Documentation/ceph-upgrade.md), including the cluster health checks described in the [health verification section](../Documentation/ceph-upgrade.md#health-verification). +Note that this sequence is modeled after the [Rook Upgrade User Guide](../Documentation/rook-upgrade.md), including the cluster health checks described in the [health verification section](../Documentation/rook-upgrade.md#health-verification). #### **Rook System Namespace** The Rook system namespace contains the single control plane for all Rook clusters in the environment. @@ -80,7 +80,7 @@ They are both managed by deployments, so the upgrade controller will update the 1. Cluster health and object/file functionality will be verified before the upgrade controller moves on to the next instances. ### **Upgrade Health Verification** -As mentioned previously, the manual health verification steps found in the [upgrade user guide](../Documentation/ceph-upgrade.md#health-verification) will be used by the upgrade controller, in an automated fashion, to ensure the cluster is healthy before proceeding with the upgrade process. +As mentioned previously, the manual health verification steps found in the [upgrade user guide](../Documentation/rook-upgrade.md#health-verification) will be used by the upgrade controller, in an automated fashion, to ensure the cluster is healthy before proceeding with the upgrade process. This approach of upgrading one component, verifying health and stability, then upgrading the next component can be viewed as a form of [canary deployment](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments). Here is a quick summary of the standard health checks the upgrade controller should perform: @@ -160,7 +160,7 @@ Listed below are a few techniques for accessing debugging artifacts from pods th * All health check status and output it encountered ## **Next Steps** -We have demonstrated that Rook is upgradable with the manual process outlined in the [Rook Upgrade User Guide](../Documentation/ceph-upgrade.md). +We have demonstrated that Rook is upgradable with the manual process outlined in the [Rook Upgrade User Guide](../Documentation/rook-upgrade.md). Fully automated upgrade support has been described within this design proposal, but will likely need to be implemented in an iterative process, with lessons learned along the way from pre-production field experience. The next step will be to implement the happy path where the upgrade controller automatically updates all Rook components in the [described sequence](#general-sequence) and stops immediately if any health checks fail and the cluster does not return to a healthy functional state. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000000..2be1a0dac11b --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,116 @@ +site_name: Rook Ceph Documentation +docs_dir: Documentation/ +site_url: 'https://rook.io' +repo_url: https://github.com/rook/rook +edit_uri: edit/master/Documentation/ +site_author: Rook Authors +site_description: "Rook Ceph Documentation" +use_directory_urls: true +copyright: | + +

+ © Rook Authors 2022. Documentation distributed under + CC-BY-4.0. +

+

+ © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has + registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our + Trademark Usage page. +

+theme: + name: material + custom_dir: .docs/overrides/ + font: false + favicon: https://rook.io/images/favicon_192x192.png + logo: https://rook.io/images/rook-logo.svg + palette: + - scheme: 'default' + primary: 'rook-blue' + accent: 'deep orange' + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode + - scheme: 'slate' + primary: 'rook-blue' + accent: 'red' + toggle: + icon: material/toggle-switch + name: Switch to light mode + icon: + repo: fontawesome/brands/github + features: + - content.tabs.link + - instant + - navigation.expand + - navigation.tabs + - navigation.tabs.sticky + - navigation.top + - navigation.tracking + - search.highlight + - search.share + - search.suggest + - tabs +extra_css: + - stylesheets/extra.css +plugins: + - search + - awesome-pages + - macros: + module_name: .docs/macros/includes/main + - minify: + minify_html: true + minify_js: true + htmlmin_opts: + remove_comments: true + #js_files: [] + - redirects: + redirect_maps: + README.md: Getting-Started/intro.md + - mike: + # these fields are all optional; the defaults are as below... + version_selector: true # set to false to leave out the version selector + css_dir: css # the directory to put the version selector's CSS + javascript_dir: js # the directory to put the version selector's JS + canonical_version: null # the version for ; `null` + # uses the version specified via `mike deploy` +markdown_extensions: + - admonition + - attr_list + - def_list + - footnotes + - meta + - toc: + permalink: true + - tables + - pymdownx.details + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg + - pymdownx.highlight: + anchor_linenums: true + use_pygments: true + linenums: true + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.magiclink + - pymdownx.mark + - pymdownx.snippets + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.superfences + - pymdownx.tabbed +extra: + version: + provider: mike + default: latest-release + social: + - icon: fontawesome/brands/slack + link: https://slack.rook.io/ + - icon: fontawesome/brands/twitter + link: https://twitter.com/rook_io + - icon: fontawesome/solid/envelopes-bulk + link: 'https://groups.google.com/forum/#!forum/rook-dev' + - icon: fontawesome/brands/medium + link: https://blog.rook.io/ diff --git a/requirements_docs.txt b/requirements_docs.txt new file mode 100644 index 000000000000..23c4ab9280da --- /dev/null +++ b/requirements_docs.txt @@ -0,0 +1,9 @@ +mike +mkdocs +mkdocs-awesome-pages-plugin +mkdocs-macros-plugin +mkdocs-material==8.* +mkdocs-material-extensions +mkdocs-minify-plugin +mkdocs-redirects +pygit2