From 067d53c692fbb94846ac7a20636dcfc685f8099c Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Mon, 18 Sep 2023 13:13:00 -0500 Subject: [PATCH] fix: v3 compressed examples, gzip capitalization (#5142) --- .../write-data/line protocol/_index.md | 70 +++++++++++-------- .../best-practices/optimize-writes.md | 21 +++--- .../cli/influxd/inspect/export-lp.md | 2 +- .../v2/reference/release-notes/influx-cli.md | 2 +- .../api/cloud-dedicated/write-compressed.sh | 18 +++-- .../api/cloud-serverless/write-compressed.sh | 15 ++-- 6 files changed, 67 insertions(+), 61 deletions(-) diff --git a/content/influxdb/cloud-dedicated/write-data/line protocol/_index.md b/content/influxdb/cloud-dedicated/write-data/line protocol/_index.md index 9d9f35f4ea..669e7c4eff 100644 --- a/content/influxdb/cloud-dedicated/write-data/line protocol/_index.md +++ b/content/influxdb/cloud-dedicated/write-data/line protocol/_index.md @@ -121,7 +121,7 @@ After setting up InfluxDB and your project, you should have the following: - Client libraries installed for writing data to InfluxDB. The following example shows how to construct `Point` objects that follow the [example `home` schema](#example-home-schema), and then write the points as line protocol to an -InfluxDB Cloud Dedicated database. +{{% product-name %}} database. {{< tabs-wrapper >}} {{% tabs %}} @@ -156,18 +156,18 @@ npm install --save @influxdata/influxdb-client {{% tab-content %}} -1. **Optional, but recommended**: Use `venv` or `conda` to activate a virtual environment for installing and executing code--for example: +1. **Optional, but recommended**: Use [`venv`](https://docs.python.org/3/library/venv.html)) or [`conda`](https://docs.continuum.io/anaconda/install/) to activate a virtual environment for installing and executing code--for example: - Inside of your project directory, enter the following command to create and activate a virtual environment for the project: + Inside of your project directory, enter the following command using `venv` to create and activate a virtual environment for the project: ```sh python3 -m venv envs/env1 && source ./envs/env1/bin/activate ``` -2. Install the `influxdb-client` InfluxDB v2 Python library. +2. Install the [`influxdb3-python`](https://github.com/InfluxCommunity/influxdb3-python), which provides the InfluxDB `influxdb_client_3` Python client library module and also installs the [`pyarrow` package](https://arrow.apache.org/docs/python/index.html) for working with Arrow data. ```sh - pip install influxdb-client + pip install influxdb3-python ``` @@ -313,24 +313,16 @@ npm install --save @influxdata/influxdb-client ``` {{% /tab-content %}} - {{% tab-content %}} 1. Create a file for your module--for example: `write-point.py`. -2. In `write-point.py`, enter the following sample code: +2. In `write-point.py`, enter the following sample code to write data in batching mode: ```python import os - from influxdb_client import InfluxDBClient, Point - - # Instantiate a client with a configuration object - # that contains your InfluxDB URL and token. - # InfluxDB ignores the org argument, but the client requires it. - client = InfluxDBClient(url=os.getenv('INFLUX_URL'), - token=os.getenv('INFLUX_TOKEN'), - org='ignored') + from influxdb_client_3 import Point, write_client_options, WritePrecision, WriteOptions, InfluxDBError # Create an array of points with tags and fields. points = [Point("home") @@ -339,22 +331,40 @@ npm install --save @influxdata/influxdb-client .field('hum', 20.2) .field('co', 9)] - # Execute code after a successful write request. + # With batching mode, define callbacks to execute after a successful or failed write request. # Callback methods receive the configuration and data sent in the request. - def success_callback(self, data): - print(f"{data}") - print(f"WRITE FINISHED") - - # Create a write client. - # Optionally, provide callback methods to execute on request success, error, and completion. - with client.write_api(success_callback=success_callback) as write_api: - # Write the data to the database. - write_api.write(bucket='get-started', - record=points, - content_encoding="identity", - content_type="text/plain; charset=utf-8",) - # Flush the write buffer and release resources. - write_api.close() + def success(self, data: str): + print(f"Successfully wrote batch: data: {data}") + + def error(self, data: str, exception: InfluxDBError): + print(f"Failed writing batch: config: {self}, data: {data} due: {exception}") + + def retry(self, data: str, exception: InfluxDBError): + print(f"Failed retry writing batch: config: {self}, data: {data} retry: {exception}") + + # Configure options for batch writing. + write_options = WriteOptions(batch_size=500, + flush_interval=10_000, + jitter_interval=2_000, + retry_interval=5_000, + max_retries=5, + max_retry_delay=30_000, + exponential_base=2) + + # Create an options dict that sets callbacks and WriteOptions. + wco = write_client_options(success_callback=success, + error_callback=error, + retry_callback=retry, + WriteOptions=write_options) + + # Instantiate a synchronous instance of the client with your + # InfluxDB credentials and write options. + with InfluxDBClient3(host=config['INFLUX_HOST'], + token=config['INFLUX_TOKEN'], + database=config['INFLUX_DATABASE'], + write_client_options=wco) as client: + + client.write(points, write_precision='s') ``` {{% /tab-content %}} diff --git a/content/influxdb/clustered/write-data/best-practices/optimize-writes.md b/content/influxdb/clustered/write-data/best-practices/optimize-writes.md index 158fe4aa59..1bf511331e 100644 --- a/content/influxdb/clustered/write-data/best-practices/optimize-writes.md +++ b/content/influxdb/clustered/write-data/best-practices/optimize-writes.md @@ -100,23 +100,20 @@ For specific instructions, see the ### Use gzip compression with the InfluxDB API When using the InfluxDB API `/api/v2/write` endpoint to write data, -compress the data with `gzip` and set the `Content-Encoding` header to `gzip`. +compress the data with `gzip` and set the `Content-Encoding` header to `gzip`--for example: {{% influxdb/custom-timestamps %}} {{% code-callout "Content-Encoding: gzip" "orange" %}} -```sh -curl --request POST "https://{{< influxdb/host >}}/api/v2/write" \ - --header "Authorization: Token DATABASE_TOKEN" \ - --header "Content-Encoding: gzip" \ - --data-urlencode "org=ignored" \ - --data-urlencode "bucket=DATABASE_NAME" \ - --data-urlencode "precision=s" \ - --data-raw " -mem,host=host1 used_percent=23.43234543 1641024000 +```bash +echo "mem,host=host1 used_percent=23.43234543 1641024000 mem,host=host2 used_percent=26.81522361 1641027600 mem,host=host1 used_percent=22.52984738 1641031200 -mem,host=host2 used_percent=27.18294630 1641034800 -" +mem,host=host2 used_percent=27.18294630 1641034800" | gzip > system.gzip \ + +curl --request POST "https://{{< influxdb/host >}}/api/v2/write?org=ignored&bucket=DATABASE_NAME&precision=s" \ + --header "Authorization: Token DATABASE_TOKEN" \ + --header "Content-Encoding: gzip" \ + --data-binary @system.gzip ``` {{% /code-callout %}} {{% /influxdb/custom-timestamps %}} diff --git a/content/influxdb/v2/reference/cli/influxd/inspect/export-lp.md b/content/influxdb/v2/reference/cli/influxd/inspect/export-lp.md index f4233db482..ed53a677e8 100644 --- a/content/influxdb/v2/reference/cli/influxd/inspect/export-lp.md +++ b/content/influxdb/v2/reference/cli/influxd/inspect/export-lp.md @@ -22,7 +22,7 @@ influxd inspect export-lp [flags] | Flag | | Description | Input type | |:---- |:--- |:----------- |:----------:| | | `--bucket-id` | ({{< req >}}) Bucket ID | string | -| | `--compress` | Compress output with GZIP | | +| | `--compress` | Compress output with gzip | | | | `--end` | End time to export (RFC3339 format) | string | | | `--engine-path` | ({{< req >}}) Path to persistent InfluxDB engine files | string | | `-h` | `--help` | Help for the `export-lp` command. | | diff --git a/content/influxdb/v2/reference/release-notes/influx-cli.md b/content/influxdb/v2/reference/release-notes/influx-cli.md index 38a52fa057..b9e79d38ee 100644 --- a/content/influxdb/v2/reference/release-notes/influx-cli.md +++ b/content/influxdb/v2/reference/release-notes/influx-cli.md @@ -227,7 +227,7 @@ The command now uses lowerCamelCase consistently for all objects keys, matching - (InfluxDB Cloud only) Add [`buck - (InfluxDB OSS only) Updates to `backup` and `restore`: - Reimplement [`backup`](/influxdb/cloud/reference/cli/influx/backup/) to support downloading embedded SQL store from InfluxDB 2.0 or later. - - Add [`--compression`](/influxdb/v2/reference/cli/influx/backup/) flag to support GZIP compression of downloaded files. + - Add [`--compression`](/influxdb/v2/reference/cli/influx/backup/) flag to support gzip compression of downloaded files. - Reimplement `restore` to support uploading embedded SQL store from InfluxDB v2.1.x. - (InfluxDB OSS only) Add [`--password`](/influxdb/cloud/reference/cli/influx/user/password/) flag to `user password` command to allow bypassing interactive prompt. diff --git a/shared/text/api/cloud-dedicated/write-compressed.sh b/shared/text/api/cloud-dedicated/write-compressed.sh index dabcc56835..56fa7eb21a 100644 --- a/shared/text/api/cloud-dedicated/write-compressed.sh +++ b/shared/text/api/cloud-dedicated/write-compressed.sh @@ -1,12 +1,10 @@ -curl --request POST "https://cluster-id.influxdb.io/api/v2/write" \ - --header "Authorization: Token DATABASE_TOKEN" \ - --header "Content-Encoding: gzip" \ - --data-urlencode "org=ignored" \ - --data-urlencode "bucket=DATABASE_NAME" \ - --data-urlencode "precision=s" \ - --data-raw " -mem,host=host1 used_percent=23.43234543 1641024000 +echo "mem,host=host1 used_percent=23.43234543 1641024000 mem,host=host2 used_percent=26.81522361 1641027600 mem,host=host1 used_percent=22.52984738 1641031200 -mem,host=host2 used_percent=27.18294630 1641034800 -" +mem,host=host2 used_percent=27.18294630 1641034800" | gzip > system.gzip \ + +curl --request POST "https://cluster-id.influxdb.io/api/v2/write" \ + --header "Authorization: Token API_TOKEN" \ + --header "Content-Type: text/plain; charset=utf-8" \ + --header "Content-Encoding: gzip" \ + --data-binary @system.gzip diff --git a/shared/text/api/cloud-serverless/write-compressed.sh b/shared/text/api/cloud-serverless/write-compressed.sh index f6b894b0f3..66fb08818d 100644 --- a/shared/text/api/cloud-serverless/write-compressed.sh +++ b/shared/text/api/cloud-serverless/write-compressed.sh @@ -1,9 +1,10 @@ -curl --request POST "http://cloud2.influxdata.com/api/v2/write?org=ORG_NAME&bucket=BUCKET_NAME&precision=s" \ - --header "Authorization: Token API_TOKEN" \ - --header "Content-Encoding: gzip" \ - --data-raw " -mem,host=host1 used_percent=23.43234543 1641024000 +echo "mem,host=host1 used_percent=23.43234543 1641024000 mem,host=host2 used_percent=26.81522361 1641027600 mem,host=host1 used_percent=22.52984738 1641031200 -mem,host=host2 used_percent=27.18294630 1641034800 -" +mem,host=host2 used_percent=27.18294630 1641034800" | gzip > system.gzip \ + +curl --request POST "https://cloud2.influxdata.com/api/v2/write?org=ORG_NAME&bucket=BUCKET_NAME&precision=s" \ + --header "Authorization: Token API_TOKEN" \ + --header "Content-Type: text/plain; charset=utf-8" \ + --header "Content-Encoding: gzip" \ + --data-binary @system.gzip