Merge branch 'benthosdev:main' into main2

.github/workflows/docker_edge.yml

@@ -14,13 +14,13 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19.x
+        go-version: 1.20.x
         check-latest: true
 
     - name: Check Out Repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
@@ -32,7 +32,7 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: Cache Docker layers
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: /tmp/.buildx-cache
         key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/integration_test.yml

@@ -15,13 +15,13 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19.x
+        go-version: 1.20.x
         check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod

.github/workflows/release.yml

@@ -13,13 +13,13 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19.x
+        go-version: 1.20.x
         check-latest: true
 
     - name: Check Out Repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
@@ -34,7 +34,7 @@ jobs:
       run: ./resources/scripts/release_notes.sh > ./release_notes.md
 
     - name: GoReleaser
-      uses: goreleaser/goreleaser-action@v2
+      uses: goreleaser/goreleaser-action@v4
       with:
         version: latest
         args: release --release-notes=./release_notes.md --timeout 60m
@@ -48,13 +48,13 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19.x
+        go-version: 1.20.x
         check-latest: true
 
     - name: Check Out Repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
@@ -66,7 +66,7 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: Cache Docker layers
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: /tmp/.buildx-cache
         key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.18.x, 1.19.x]
+        go-version: [1.19.x, 1.20.x]
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     env:
@@ -26,9 +26,11 @@ jobs:
         check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
@@ -40,7 +42,7 @@ jobs:
           ${{ runner.os }}-go-
 
     - name: GoReleaser
-      uses: goreleaser/goreleaser-action@v2
+      uses: goreleaser/goreleaser-action@v4
       with:
         version: latest
         args: build --snapshot --timeout 60m
@@ -55,13 +57,13 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19.x
+        go-version: 1.20.x
         check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
@@ -91,21 +93,21 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19.x
+        go-version: 1.20.x
         check-latest: true
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Lint
-      uses: golangci/golangci-lint-action@v2
+      uses: golangci/golangci-lint-action@v3
       with:
         version: latest
         args: --timeout 30m cmd/... internal/... public/...
 
     # TODO: Remove this after https://github.com/golangci/golangci-lint/issues/3094 is addressed
-    - name: Get govulncheck
-      run: go install golang.org/x/vuln/cmd/govulncheck@latest
+    # - name: Get govulncheck
+    #   run: go install golang.org/x/vuln/cmd/govulncheck@latest
 
-    - name: Run govulncheck
-      run: govulncheck ./...
+    # - name: Run govulncheck
+    #   run: govulncheck ./...

.golangci.yml

@@ -35,23 +35,22 @@ linters-settings:
 linters:
   disable-all: true
   enable:
-    # Default linters reported by `golangci-lint help linters` in v1.41.1
-    # Disabled for Go 1.18
-    - gosimple
-    - staticcheck
-    - unused
+    # Default linters reported by `golangci-lint help linters` in v1.52.0
     - errcheck
+    - gosimple
     - govet
     - ineffassign
+    - staticcheck
     - typecheck
+    - unused
     # Extra linters:
-    # Disabled for Go 1.18
-    # - wastedassign
+    - wastedassign
     - stylecheck
     - gofmt
     - goimports
-    - gocritic
-    - revive
+    # gocritic is very slow (golangci-lint v1.52.0)
+    # - gocritic
+    # - revive
     - unconvert
     - durationcheck
     - depguard

CHANGELOG.md

@@ -7,8 +7,117 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- The `-e/--env-file` cli flag can now be specified multiple times.
+
+### Fixed
+
+- The `find_all` bloblang method no longer produces results that are of an `unknown` type.
+
+## 4.13.0 - 2023-03-15
+
+### Added
+
+- Fix vulnerability [GO-2023-1571](https://pkg.go.dev/vuln/GO-2023-1571)
+- New `nats_kv` processor, input and output.
+- Field `partition` added to the `kafka_franz` output, allowing for manual partitioning.
+
+### Fixed
+
+- The `broker` output with the pattern `fan_out_sequential` will no longer abandon in-flight requests that are error blocked until the full shutdown timeout has occurred.
+- Fixed a regression bug in the `sequence` input where the returned messages have type `unknown`. This issue was introduced in v4.10.0 (cefa288).
+- The `broker` input no longer reports itself as unavailable when a child input has intentionally closed.
+- Config unit tests that check for structured data should no longer fail in all cases.
+- The `http_server` input with a custom address now supports path variables.
+
+## 4.12.1 - 2023-02-23
+
+### Fixed
+
+- Fixed a regression bug in the `nats` components where panics occur during a flood of messages. This issue was introduced in v4.12.0 (45f785a).
+
+## 4.12.0 - 2023-02-20
+
+### Added
+
+- Format `csv:x` added to the `unarchive` processor.
+- Field `max_buffer` added to the `aws_s3` input.
+- Field `open_message_type` added to the `websocket` input.
+- The experimental `--watcher` cli flag now takes into account file deletions and new files that match wildcard patterns.
+- Field `dump_request_log_level` added to HTTP components.
+- New `couchbase` cache implementation.
+- New `compress` and `decompress` Bloblang methods.
+- Field `endpoint` added to the `gcp_pubsub` input and output.
+- Fields `file_name`, `file_extension` and `request_id` added to the `snowflake_put` output.
+- Add interpolation support to the `path` field of the `snowflake_put` output.
+- Add ZSTD compression support to the `compression` field of the `snowflake_put` output.
+- New Bloblang method `concat`.
+- New `redis` ratelimit.
+- The `socket_server` input now supports `tls` as a network type.
+- New bloblang function `timestamp_unix_milli`.
+- New bloblang method `ts_unix_milli`.
+- JWT based HTTP authentication now supports `EdDSA`.
+- New `flow_control` fields added to the `gcp_pubsub` output.
+- Added bloblang methods `sign_jwt_hs256`, `sign_jwt_hs384` and `sign_jwt_hs512`
+- New bloblang methods `parse_jwt_hs256`, `parse_jwt_hs384`, `parse_jwt_hs512`.
+- The `open_telemetry_collector` tracer now automatically sets the `service.name` and `service.version` tags if they are not configured by the user.
+- New bloblang string methods `trim_prefix` and `trim_suffix`.
+
+### Fixed
+
+- Fixed an issue where messages caught in a retry loop from inputs that do not support nacks (`generate`, `kafka`, `file`, etc) could be retried in their post-mutation form from the `switch` output rather than the original copy of the message.
+- The `sqlite` buffer should no longer print `Failed to ack buffer message` logs during graceful termination.
+- The default value of the `conn_max_idle` field has been changed from 0 to 2 for all `sql_*` components in accordance
+to the [`database/sql` docs](https://pkg.go.dev/database/sql#DB.SetMaxIdleConns).
+- The `parse_csv` bloblang method with `parse_header_row` set to `false` no longer produces rows that are of an `unknown` type.
+- Fixed a bug where the `oracle` driver for the `sql_*` components was returning timestamps which were getting marshalled into an empty JSON object instead of a string.
+- The `aws_sqs` input no longer backs off on subsequent empty requests when long polling is enabled.
+- It's now possible to mock resources within the main test target file in config unit tests.
+- Unit test linting no longer incorrectly expects the `json_contains` predicate to contain a string value only.
+- Config component initialisation errors should no longer show nested path annotations.
+- Prevented panics from the `jq` processor when querying invalid types.
+- The `jaeger` tracer no longer emits the `service.version` tag automatically if the user sets the `service.name` tag explicitly.
+- The `int64()`, `int32()`, `uint64()` and `uint32()` bloblang methods can now infer the number base as documented [here](https://pkg.go.dev/strconv#ParseInt).
+- The `mapping` and `mutation` processors should provide metrics and tracing events again.
+- Fixed a data race in the `redis_streams` input.
+- Upgraded the Redis components to `github.com/redis/go-redis/v9`.
+
+## 4.11.0 - 2022-12-21
+
+### Added
+
 - Field `default_encoding` added to the `parquet_encode` processor.
 - Field `client_session_keep_alive` added to the `snowflake_put` output.
+- Bloblang now supports metadata access via `@foo` syntax, which also supports arbitrary values.
+- TLS client certs now support both PKCS#1 and PKCS#8 encrypted keys.
+- New `redis_script` processor.
+- New `wasm` processor.
+- Fields marked as secrets will no longer be printed with `benthos echo` or debug HTTP endpoints.
+- Add `no_indent` parameter to the `format_json` bloblang method.
+- New `format_xml` bloblang method.
+- New `batched` higher level input type.
+- The `gcp_pubsub` input now supports optionally creating subscriptions.
+- New `sqlite` buffer.
+- Bloblang now has `int64`, `int32`, `uint64` and `uint32` methods for casting explicit integer types.
+- Field `application_properties_map` added to the `amqp1` output.
+- Param `parse_header_row`, `delimiter` and `lazy_quotes` added to the `parse_csv` bloblang method.
+- Field `delete_on_finish` added to the `csv` input.
+- Metadata fields `header`, `path`, `mod_time_unix` and `mod_time` added to the `csv` input.
+- New `couchbase` processor.
+- Field `max_attempts` added to the `nsq` input.
+- Messages consumed by the `nsq` input are now enriched with metadata.
+- New Bloblang method `parse_url`.
+
+### Fixed
+
+- Fixed a regression bug in the `mongodb` processor where message errors were not set any more. This issue was introduced in v4.7.0 (64eb72).
+- The `avro-ocf:marshaler=json` input codec now omits unexpected logical type fields.
+- Fixed a bug in the `sql_insert` output (see commit c6a71e9) where transaction-based drivers (`clickhouse` and `oracle`) would fail to roll back an in-progress transaction if any of the messages caused an error.
+- The `resource` input should no longer block the first layer of graceful termination.
+
+### Changed
+
+- The `catch` method now defines the context of argument mappings to be the string of the caught error. In previous cases the context was undocumented, vague and would often bind to the outer context. It's still possible to reference this outer context by capturing the error (e.g. `.catch(_ -> this)`).
+- Field interpolations that fail due to mapping errors will no longer produce placeholder values and will instead provide proper errors that result in nacks or retries similar to other issues.
 
 ## 4.10.0 - 2022-10-26
 

Makefile

@@ -2,6 +2,7 @@
 
 TAGS ?=
 
+GOMAXPROCS         ?= 1
 INSTALL_DIR        ?= $(GOPATH)/bin
 WEBSITE_DIR        ?= ./website
 DEST_DIR           ?= ./target
@@ -91,7 +92,7 @@ fmt:
 
 lint:
 	@go vet $(GO_FLAGS) ./...
-	@golangci-lint run --timeout 5m cmd/... internal/... public/...
+	@golangci-lint -j $(GOMAXPROCS) run --timeout 5m cmd/... internal/... public/...
 
 test: $(APPS)
 	@go test $(GO_FLAGS) -ldflags "$(LD_FLAGS)" -timeout 3m ./...

cmd/tools/benthos_docs_gen/bloblang_test.go

@@ -75,6 +75,7 @@ func TestMethodExamples(t *testing.T) {
 	})
 
 	_, err = tmpJSONFile.WriteString(`
+{
   "type":"object",
   "properties":{
     "foo":{

config/test/infile_resource_mock.yaml

@@ -0,0 +1,23 @@
+pipeline:
+  processors:
+    - resource: http_submit
+
+processor_resources:
+  - label: http_submit
+    http:
+      url: http://nonexistant.foo/
+      verb: POST
+
+tests:
+  - name: test_case
+    target_processors: /pipeline/processors
+    mocks:
+      http_submit:
+        mapping: 'root = {"abc": 123}'
+    input_batch:
+      - json_content:
+          foo: bar
+    output_batches:
+      - - json_equals:
+            abc: 123
+          bloblang: '!errored()'

config/test/json_contains_predicate.yaml

@@ -0,0 +1,16 @@
+processor_resources:
+  - label: woof_drop
+    mapping: |
+      root = if this.resource."service.name" == "woof" { deleted() }
+
+tests:
+  - name: woof drop test
+    target_processors: 'woof_drop'
+    input_batch:
+      - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"aaa","service.name":"meow"}}'
+      - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"bbb","service.name":"woof"}}'
+      - content: '{"resource":{"cloud.platform":"aws_eks","host.id":"ccc","service.name":"quack"}}'
+    output_batches:
+      -
+        - json_contains: { "resource": { "cloud.platform": "aws_eks", "host.id": "aaa" } }
+        - json_contains: { "resource": { "cloud.platform": "aws_eks", "host.id": "ccc" } }