Merge pull request influxdata#302 from influxdata/cloud-to-oss-migration

Make cloud migration template more generic

influxdb-cloud-oss-migration/migration.yml

@@ -1,29 +1,29 @@
 apiVersion: influxdata.com/v2alpha1
 kind: Label
 metadata:
-    name: vigilant-goldwasser-524001
+    name: romantic-colden-124001
 spec:
     color: '#00a3ff'
-    name: Cloud to OSS migration
+    name: Cloud migration
 ---
 apiVersion: influxdata.com/v2alpha1
 kind: Bucket
 metadata:
-    name: strange-matsumoto-924003
+    name: lucid-bohr-124005
 spec:
     associations:
       - kind: Label
-        name: vigilant-goldwasser-524001
+        name: romantic-colden-124001
     name: migration
 ---
 apiVersion: influxdata.com/v2alpha1
 kind: Task
 metadata:
-    name: dreamy-tharp-924007
+    name: tender-buck-124009
 spec:
     associations:
       - kind: Label
-        name: vigilant-goldwasser-524001
+        name: romantic-colden-124001
     every: 5m
     name: Migrate data from InfluxDB Cloud
     query: |-
@@ -37,27 +37,28 @@ spec:
         // Configure the migration
         migration = {
             start: 2022-01-01T00:00:00Z,
-            stop: 2022-01-02T00:00:00Z,
+            stop: 2022-02-01T00:00:00Z,
             batchInterval: 1h,
             batchBucket: "migration",
-            cloudHost: "https://cloud2.influxdata.com",
-            cloudOrg: "example-cloud-org",
-            cloudToken: secrets.get(key: "INFLUXDB_CLOUD_TOKEN"),
-            cloudBucket: "example-cloud-bucket",
-            ossBucket: "example-oss-bucket",
+            sourceHost: "https://cloud2.influxdata.com",
+            sourceOrg: "example-cloud-org",
+            sourceToken: secrets.get(key: "INFLUXDB_CLOUD_TOKEN"),
+            sourceBucket: "example-cloud-bucket",
+            destinationBucket: "example-oss-bucket",
         }
 
         // batchRange dynamically returns a record with start and stop properties for
-        // the current batch based on the stop time of the previous batch stored in the
-        // migration.batchBucket for the specific migration.cloudOrg and
-        // migration.cloudBucket.
+        // the current batch. It queries migration metadata stored in the
+        // `migration.batchBucket` to determine the stop time of the previous batch.
+        // It uses the previous stop time as the new start time for the current batch
+        // and adds the `migration.batchInterval` to determine the current batch stop time.
         batchRange = () => {
             _lastBatchStop =
                 (from(bucket: migration.batchBucket)
                     |> range(start: migration.start)
                     |> filter(fn: (r) => r._field == "batch_stop")
-                    |> filter(fn: (r) => r.srcOrg == migration.cloudOrg)
-                    |> filter(fn: (r) => r.srcBucket == migration.cloudBucket)
+                    |> filter(fn: (r) => r.srcOrg == migration.sourceOrg)
+                    |> filter(fn: (r) => r.srcBucket == migration.sourceBucket)
                     |> last()
                     |> findRecord(fn: (key) => true, idx: 0))._value
             _batchStart =
@@ -80,12 +81,16 @@ spec:
             else
                 "Migration in progress"
 
-        // Query all data from the from the specified InfluxDB Cloud bucket within the
-        // batch-defined time range. To limit migrated data by measurement, tag, or
-        // field, add a `filter()` function after `range()` with the appropriate
-        // predicate fn.
+        // Query all data from the specified source bucket within the batch-defined time
+        // range. To limit migrated data by measurement, tag, or field, add a `filter()`
+        // function after `range()` with the appropriate predicate fn.
         data = () =>
-            from(host: migration.cloudHost, org: migration.cloudOrg, token: migration.cloudToken, bucket: migration.cloudBucket)
+            from(
+                host: migration.sourceHost,
+                org: migration.sourceOrg,
+                token: migration.sourceToken,
+                bucket: migration.sourceBucket,
+            )
                 |> range(start: batch.start, stop: batch.stop)
 
         // rowCount is a stream of tables that contains the number of rows returned in
@@ -96,7 +101,7 @@ spec:
                 |> count()
 
         // emptyRange is a stream of tables that acts as filler data if the batch is
-        // emtpy. This is used to generate batch metadata for empty batches and is
+        // empty. This is used to generate batch metadata for empty batches and is
         // necessary to correctly increment the time range for the next batch.
         emptyRange = array.from(rows: [{_start: batch.start, _stop: batch.stop, _value: 0}])
 
@@ -115,9 +120,9 @@ spec:
                             ({
                                 _time: now(),
                                 _measurement: "batches",
-                                srcOrg: migration.cloudOrg,
-                                srcBucket: migration.cloudBucket,
-                                dstBucket: migration.ossBucket,
+                                srcOrg: migration.sourceOrg,
+                                srcBucket: migration.sourceBucket,
+                                dstBucket: migration.destinationBucket,
                                 batch_start: string(v: batch.start),
                                 batch_stop: string(v: batch.stop),
                                 rows: r._value,
@@ -132,7 +137,7 @@ spec:
 
         // Write the queried data to the specified InfluxDB OSS bucket.
         data()
-            |> to(bucket: migration.ossBucket)
+            |> to(bucket: migration.destinationBucket)
 
         // Generate and store batch metadata in the migration.batchBucket.
         metadata()
@@ -141,11 +146,11 @@ spec:
 apiVersion: influxdata.com/v2alpha1
 kind: Variable
 metadata:
-    name: earnest-yalow-92400b
+    name: earnest-poincare-12400d
 spec:
     associations:
       - kind: Label
-        name: vigilant-goldwasser-524001
+        name: romantic-colden-124001
     language: flux
     name: migrationTaskID
     query: |-
@@ -157,45 +162,45 @@ spec:
 apiVersion: influxdata.com/v2alpha1
 kind: Variable
 metadata:
-    name: hopeful-mendeleev-924013
+    name: earning-haslett-124011
 spec:
     associations:
       - kind: Label
-        name: vigilant-goldwasser-524001
+        name: romantic-colden-124001
     language: flux
-    name: source_org
+    name: source_bucket
     query: |-
         import "influxdata/influxdb/schema"
 
-        schema.tagValues(bucket: "migration", tag: "srcOrg")
+        schema.tagValues(bucket: "migration", tag: "srcBucket")
+    selected:
+      - docs-metrics
     type: query
 ---
 apiVersion: influxdata.com/v2alpha1
 kind: Variable
 metadata:
-    name: recursing-mendeleev-92400f
+    name: zealous-cartwright-124015
 spec:
     associations:
       - kind: Label
-        name: vigilant-goldwasser-524001
+        name: romantic-colden-124001
     language: flux
-    name: source_bucket
+    name: source_org
     query: |-
         import "influxdata/influxdb/schema"
 
-        schema.tagValues(bucket: "migration", tag: "srcBucket")
-    selected:
-      - docs-metrics
+        schema.tagValues(bucket: "migration", tag: "srcOrg")
     type: query
 ---
 apiVersion: influxdata.com/v2alpha1
 kind: Dashboard
 metadata:
-    name: nice-shockley-d24001
+    name: angry-panini-924001
 spec:
     associations:
       - kind: Label
-        name: vigilant-goldwasser-524001
+        name: romantic-colden-124001
     charts:
       - axes:
           - base: "10"
@@ -379,6 +384,5 @@ spec:
         suffix: '%'
         width: 4
         xPos: 8
-    description: Monitor the progress of data migrations from InfluxDB Cloud to InfluxDB
-        OSS
-    name: InfluxDB Cloud to OSS Migration Progress
+    description: Monitor the progress of data migrations from InfluxDB Cloud
+    name: InfluxDB Cloud Migration Progress

influxdb-cloud-oss-migration/readme.md

@@ -1,14 +1,15 @@
-## InfluxDB Cloud to OSS Migration Template
+## InfluxDB Cloud Migration Template
 
 Provided by: [Scott Anderson](https://github.com/sanderson/)
 
-Use this InfluxDB Cloud to OSS Migration template to run and monitor data
-migrations from [InfluxDB Cloud]() to [InfluxDB OSS]().
+Use this InfluxDB Cloud Migration template to run and monitor data migrations
+from [InfluxDB Cloud](https://cloud2.influxdata.com) to [InfluxDB OSS](https://www.influxdata.com/)
+or another InfluxDB Cloud organization.
 
-![InfluxDB Cloud to OSS Migration Dashboard Screenshot](img/migration-dashboard.png)
+![InfluxDB Cloud Migration Dashboard Screenshot](img/migration-dashboard.png)
 
-**Imporant:** Install this template on the **InfluxDB OSS** instance you would
-like to migrate data to.
+**Important:** Install this template on the **InfluxDB OSS** or **InfluxDB Cloud**
+instance you want to migrate data to.
 
 ## Quick Install
 
@@ -31,26 +32,27 @@ influx apply -u https://raw.githubusercontent.com/influxdata/community-templates
 ## Included Resources
 
 - 1 Bucket: `migration`
-- 1 Dashboard: `InfluxDB Cloud to OSS Migration Progress`
+- 1 Dashboard: `InfluxDB Cloud Migration Progress`
 - 3 Dashboard Variables:
   - `source_org`
   - `source_bucket`
   - `migrationTaskID`
 - 1 Task: `Migrate data from InfluxDB Cloud`
-- 1 Label: `Clout to OSS migration`
+- 1 Label: `Cloud migration`
 
 ## Setup Instructions
 
 General instructions on using InfluxDB Templates can be found in [Use a template](../docs/use_a_template.md).
 
 ### Set up the migration
 
-1.  **In InfluxDB Cloud**, [create an API token](https://docs.influxdata.com/influxdb/cloud/security/tokens/create-token/)
-        with **read access** to the bucket you want to migrate.
+1.  **In the InfluxDB Cloud instance you want to migrate data from**,
+    [create an API token](https://docs.influxdata.com/influxdb/cloud/security/tokens/create-token/)
+    with **read access** to the bucket you want to migrate.
 
-2.  **In InfluxDB OSS**:
-    1.  Add your **InfluxDB Cloud API token** as a secret using the key,
-        `INFLUXDB_CLOUD_TOKEN`.
+2.  **In the InfluxDB Cloud or OSS instance you want to migrate data to**:
+    1.  Add the **InfluxDB Cloud API token from the source organization** as a
+        secret using the key, `INFLUXDB_CLOUD_TOKEN`.
         _See [Add secrets](https://docs.influxdata.com/influxdb/latest/security/secrets/add/) for more information._
     2.  [Create a bucket](https://docs.influxdata.com/influxdb/latest/organizations/buckets/create-bucket/)
         **to migrate data to**.
@@ -65,14 +67,14 @@ General instructions on using InfluxDB Templates can be found in [Use a template
         - **stop**: Latest time to include in the migration.
         - **batchInterval**: Duration of each time-based batch.
           _See [Determine your batch interval](https://docs.influxdata.com/influxdb/latest/write-data/migrate-cloud-to-oss/#determine-your-batch-interval)._
-        - **batchBucket**: InfluxDB OSS bucket to store migration batch metadata in.
-        - **cloudHost**: [InfluxDB Cloud region URL](https://docs.influxdata.com/influxdb/cloud/reference/regions)
+        - **batchBucket**: InfluxDB bucket to store migration batch metadata in.
+        - **sourceHost**: [InfluxDB Cloud region URL](https://docs.influxdata.com/influxdb/cloud/reference/regions)
           to migrate data from.
-        - **cloudOrg**: InfluxDB Cloud organization to migrated data from.
-        - **cloudToken**: InfluxDB Cloud API token. To keep the API token secure, store
+        - **sourceOrg**: InfluxDB Cloud organization to migrate data from.
+        - **sourceToken**: InfluxDB Cloud API token. To keep the API token secure, store
           it as a secret in InfluxDB OSS.
-        - **cloudBucket**: InfluxDB Cloud bucket to migrate data from.
-        - **ossBucket**: InfluxDB OSS bucket to migrate data to.
+        - **sourceBucket**: InfluxDB Cloud bucket to migrate data from.
+        - **destinationBucket**: InfluxDB bucket to migrate data to.
 
     5. Save and enable the task to begin the migration.
 
@@ -84,8 +86,10 @@ error exhausting result iterator: error calling function "die" @41:9-41:86:
 Batch range is beyond the migration range. Migration is complete.
 ```
 
-For detailed information about this data migration process, see
-[Migrate data from InfluxDB Cloud to InfluxDB OSS](https://docs.influxdata.com/influxdb/latest/write-data/migrate-cloud-to-oss/).
+For detailed information about this data migration process, see one of the following:
+
+- [Migrate data from InfluxDB Cloud to InfluxDB OSS](https://docs.influxdata.com/influxdb/latest/migrate-data/migrate-cloud-to-oss/).
+- [Migrate data between InfluxDB Cloud organizations](https://docs.influxdata.com/influxdb/cloud/migrate-data/migrate-cloud-to-cloud/).
 
 ## Contact