Merge branch 'main' into bloblang-ulid-func

.github/workflows/test.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   cross-build:
-    if: ${{ github.repository == 'benthosdev/benthos' || github.event_name != 'schedule' }}
+    if: ${{ github.repository == 'benthosdev/benthos' && github.event_name != 'schedule' }}
     strategy:
       fail-fast: false
       matrix:

CHANGELOG.md

@@ -7,13 +7,64 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
-- The `-e/--env-file` cli flag can now be specified multiple times.
+- Fields `auth.user_jwt` and `auth.user_nkey_seed` added to all NATS components.
 - bloblang: added `ulid(encoding, random_source)` function to generate Universally Unique Lexicographically Sortable Identifiers (ULIDs).
 
+## 4.15.0 - 2023-05-05
+
+### Added
+
+- Flag `--skip-env-var-check` added to the `lint` subcommand, this disables the new linting behaviour where environment variable interpolations without defaults throw linting errors when the variable is not defined.
+- The `kafka_franz` input now supports explicit partitions in the field `topics`.
+- The `kafka_franz` input now supports batching.
+- New `metadata` Bloblang function for batch-aware structured metadata queries.
+- Go API: Running the Benthos CLI with a context set with a deadline now triggers graceful termination before the deadline is reached.
+- Go API: New `public/service/servicetest` package added for functions useful for testing custom Benthos builds.
+- New `lru` and `ttlru` in-memory caches.
+
+### Fixed
+
+- Provide msgpack plugins through `public/components/msgpack`.
+- The `kafka_franz` input should no longer commit offsets one behind the next during partition yielding.
+- The streams mode HTTP API should no longer route requests to `/streams/<stream-ID>` to the `/streams` handler. This issue was introduced in v4.14.0.
+
+## 4.14.0 - 2023-04-25
+
+### Added
+
+- The `-e/--env-file` cli flag can now be specified multiple times.
+- New `studio pull` cli subcommand for running [Benthos Studio](https://studio.benthos.dev) session deployments.
+- Metadata field `kafka_tombstone_message` added to the `kafka` and `kafka_franz` inputs.
+- Method `SetEnvVarLookupFunc` added to the stream builder API.
+- The `discord` input and output now use the official chat client API and no longer rely on poll-based HTTP requests, this should result in more efficient and less erroneous behaviour.
+- New bloblang timestamp methods `ts_add_iso8601` and `ts_sub_iso8601`.
+- All SQL components now support the `trino` driver.
+- New input codec `csv-safe`.
+- Added `base64rawurl` scheme to both the `encode` and `decode` Bloblang methods.
+- New `find_by` and `find_all_by` Bloblang methods.
+- New `skipbom` input codec.
+- New `javascript` processor.
+
 ### Fixed
 
 - The `find_all` bloblang method no longer produces results that are of an `unknown` type.
+- The `find_all` and `find` Bloblang methods no longer fail when the value argument is a field reference.
 - Endpoints specified by HTTP server components using both the general `http` server block or their own custom server addresses should now be treated as path prefixes. This corrects a behavioural change that was introduced when both respective server options were updated to support path parameters.
+- Prevented a panic caused when using the `encrypt_aes` and `decrypt_aes` Bloblang methods with a mismatched key/iv lengths.
+- The `snowpipe` field of the `snowflake_put` output can now be omitted from the config without raising an error.
+- Batch-aware processors such as `mapping` and `mutation` should now report correct error metrics.
+- Running `benthos blobl server` should no longer panic when a mapping with variable read/writes is executed in parallel.
+- Speculative fix for the `cloudwatch` metrics exporter rejecting metrics due to `minimum field size of 1, PutMetricDataInput.MetricData[0].Dimensions[0].Value`.
+- The `snowflake_put` output now prevents silent failures under certain conditions. Details [here](https://github.com/snowflakedb/gosnowflake/issues/701).
+- Reduced the amount of pre-compilation of Bloblang based linting rules for documentation fields, this should dramatically improve the start up time of Benthos (~1s down to ~200ms).
+- Environment variable interpolations with an empty fallback (`${FOO:}`) are now valid.
+- Fixed an issue where the `mongodb` output wasn't using bulk send requests according to batching policies.
+- The `amqp_1` input now falls back to accessing `Message.Value` when the data is empty.
+
+### Changed
+
+- When a config contains environment variable interpolations without a default value (i.e. `${FOO}`), if that environment variable is not defined a linting error will be emitted. Shutting down due to linting errors can be disabled with the `--chilled` cli flag, and variables can be specified with an empty default value (`${FOO:}`) in order to make the previous behaviour explicit and prevent the new linting error.
+- The `find` and `find_all` Bloblang methods no longer support query arguments as they were incompatible with supporting value arguments. For query based arguments use the new `find_by` and `find_all_by` methods.
 
 ## 4.13.0 - 2023-03-15
 

Makefile

@@ -39,25 +39,25 @@ deps:
 	@go mod tidy
 
 SOURCE_FILES = $(shell find internal public cmd -type f)
-TEMPLATE_FILES = $(shell find template -path template/test -prune -o -type f -name "*.yaml")
+TEMPLATE_FILES = $(shell find internal/impl -type f -name "template_*.yaml")
 
-$(PATHINSTBIN)/%: $(SOURCE_FILES) $(TEMPLATE_FILES)
+$(PATHINSTBIN)/%: $(SOURCE_FILES)
 	@go build $(GO_FLAGS) -tags "$(TAGS)" -ldflags "$(LD_FLAGS) $(VER_FLAGS)" -o $@ ./cmd/$*
 
 $(APPS): %: $(PATHINSTBIN)/%
 
 TOOLS = benthos_docs_gen
 tools: $(TOOLS)
 
-$(PATHINSTTOOLS)/%: $(SOURCE_FILES) $(TEMPLATE_FILES)
+$(PATHINSTTOOLS)/%: $(SOURCE_FILES)
 	@go build $(GO_FLAGS) -tags "$(TAGS)" -ldflags "$(LD_FLAGS) $(VER_FLAGS)" -o $@ ./cmd/tools/$*
 
 $(TOOLS): %: $(PATHINSTTOOLS)/%
 
 SERVERLESS = benthos-lambda
 serverless: $(SERVERLESS)
 
-$(PATHINSTSERVERLESS)/%: $(SOURCE_FILES) $(TEMPLATE_FILES)
+$(PATHINSTSERVERLESS)/%: $(SOURCE_FILES)
 	@CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
 		go build $(GO_FLAGS) -tags "$(TAGS)" -ldflags "$(LD_FLAGS) $(VER_FLAGS)" -o $@ ./cmd/serverless/$*
 	@zip -m -j $@.zip $@
@@ -92,7 +92,7 @@ lint:
 
 test: $(APPS)
 	@go test $(GO_FLAGS) -ldflags "$(LD_FLAGS)" -timeout 3m ./...
-	@$(PATHINSTBIN)/benthos template lint ./template/...
+	@$(PATHINSTBIN)/benthos template lint $(TEMPLATE_FILES)
 	@$(PATHINSTBIN)/benthos test ./config/test/...
 
 test-race: $(APPS)
@@ -111,6 +111,7 @@ clean:
 
 docs: $(APPS) $(TOOLS)
 	@$(PATHINSTTOOLS)/benthos_docs_gen $(DOCS_FLAGS)
-	@$(PATHINSTBIN)/benthos lint --deprecated "./config/**/*.yaml" \
+	@$(PATHINSTBIN)/benthos lint --deprecated "./config/examples/*.yaml" \
 		"$(WEBSITE_DIR)/cookbooks/**/*.md" \
 		"$(WEBSITE_DIR)/docs/**/*.md"
+	@$(PATHINSTBIN)/benthos template lint "./config/template_examples/*.yaml"

cmd/benthos/main.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"context"
+
 	"github.com/benthosdev/benthos/v4/internal/cli"
 
 	// Import all plugins defined within the repo.
 	_ "github.com/benthosdev/benthos/v4/public/components/all"
 )
 
 func main() {
-	cli.Run()
+	cli.Run(context.Background())
 }

cmd/tools/benthos_docs_gen/main.go

@@ -35,10 +35,6 @@ func main() {
 	flag.StringVar(&docsDir, "dir", docsDir, "The directory to write docs to")
 	flag.Parse()
 
-	if err := template.InitNativeTemplates(); err != nil {
-		panic(err)
-	}
-
 	service.GlobalEnvironment().WalkInputs(viewForDir(path.Join(docsDir, "./inputs")))
 	service.GlobalEnvironment().WalkBuffers(viewForDir(path.Join(docsDir, "./buffers")))
 	service.GlobalEnvironment().WalkCaches(viewForDir(path.Join(docsDir, "./caches")))

config/examples/discord_bot.yaml

@@ -1,11 +1,9 @@
 input:
   discord:
-    poll_period: 3s
-    channel_id: ${DISCORD_CHANNEL}
-    bot_token: ${DISCORD_BOT_TOKEN}
+    channel_id: ${DISCORD_CHANNEL:xxx}
+    bot_token: ${DISCORD_BOT_TOKEN:xxx}
     cache: request_tracking
     cache_key: last_message_received
-    limit: 10
 
 pipeline:
   processors:
@@ -67,8 +65,8 @@ pipeline:
 
 output:
   discord:
-    channel_id: ${DISCORD_CHANNEL}
-    bot_token: ${DISCORD_BOT_TOKEN}
+    channel_id: ${DISCORD_CHANNEL:xxx}
+    bot_token: ${DISCORD_BOT_TOKEN:xxx}
 
 cache_resources:
   - label: request_tracking

go.mod

@@ -1,12 +1,14 @@
 module github.com/benthosdev/benthos/v4
 
+replace github.com/99designs/keyring => github.com/Jeffail/keyring v1.2.3
+
 require (
 	cloud.google.com/go/bigquery v1.44.0
 	cloud.google.com/go/pubsub v1.27.1
 	cloud.google.com/go/storage v1.28.0
 	cuelang.org/go v0.4.2
-	github.com/Azure/azure-sdk-for-go v61.1.0+incompatible
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
+	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0
 	github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1
 	github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd
 	github.com/Azure/go-amqp v0.17.0
@@ -26,21 +28,24 @@ require (
 	github.com/beanstalkd/go-beanstalk v0.2.0
 	github.com/benhoyt/goawk v1.21.0
 	github.com/bradfitz/gomemcache v0.0.0-20230124162541-5f7a7d875746
+	github.com/bwmarrin/discordgo v0.27.1
 	github.com/bwmarrin/snowflake v0.3.0
-	github.com/bxcodec/faker/v3 v3.8.1
 	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/clbanning/mxj/v2 v2.5.7
 	github.com/colinmarc/hdfs v1.1.3
 	github.com/couchbase/gocb/v2 v2.6.0
 	github.com/denisenkom/go-mssqldb v0.12.3
 	github.com/dgraph-io/ristretto v0.1.1
+	github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6
+	github.com/dop251/goja_nodejs v0.0.0-20220808115320-bac29516aae9
 	github.com/dustin/go-humanize v1.0.1
 	github.com/eclipse/paho.mqtt.golang v1.4.2
 	github.com/fatih/color v1.14.1
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/generikvault/gvalstrings v0.0.0-20180926130504-471f38f0112a
+	github.com/go-faker/faker/v4 v4.1.0
 	github.com/go-sql-driver/mysql v1.6.0
-	github.com/gocql/gocql v1.3.1
+	github.com/gocql/gocql v1.4.0
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang-jwt/jwt/v4 v4.4.3
@@ -50,6 +55,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/gosimple/slug v1.13.1
+	github.com/hashicorp/golang-lru/v2 v2.0.1
 	github.com/influxdata/go-syslog/v3 v3.0.0
 	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
 	github.com/itchyny/gojq v0.12.11
@@ -61,15 +67,15 @@ require (
 	github.com/linkedin/goavro/v2 v2.12.0
 	github.com/matoous/go-nanoid/v2 v2.0.0
 	github.com/microcosm-cc/bluemonday v1.0.22
-	github.com/mitchellh/mapstructure v1.4.3
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/nats-io/nats.go v1.23.0
 	github.com/nats-io/nkeys v0.3.0
 	github.com/nats-io/stan.go v0.10.2
 	github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
 	github.com/nsqio/go-nsq v1.1.0
 	github.com/oklog/ulid v1.3.1
 	github.com/olivere/elastic/v7 v7.0.31
-	github.com/ory/dockertest/v3 v3.8.1
+	github.com/ory/dockertest/v3 v3.9.1
 	github.com/oschwald/geoip2-golang v1.5.0
 	github.com/pebbe/zmq4 v1.2.7
 	github.com/pierrec/lz4/v4 v4.1.17
@@ -90,8 +96,9 @@ require (
 	github.com/smira/go-statsd v1.3.2
 	github.com/snowflakedb/gosnowflake v1.6.16
 	github.com/stretchr/testify v1.8.1
-	github.com/tetratelabs/wazero v1.0.0-rc.1
+	github.com/tetratelabs/wazero v1.0.1
 	github.com/tilinna/z85 v1.0.0
+	github.com/trinodb/trino-go-client v0.309.0
 	github.com/twmb/franz-go v1.13.0
 	github.com/twmb/franz-go/pkg/kmsg v1.4.0
 	github.com/urfave/cli/v2 v2.11.0
@@ -122,6 +129,7 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v3 v3.0.1
 	modernc.org/sqlite v1.19.1
+	zvelo.io/ttlru v1.0.10
 )
 
 require (
@@ -134,7 +142,7 @@ require (
 	github.com/99designs/keyring v1.2.1 // indirect
 	github.com/AthenZ/athenz v1.10.43 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/Azure/azure-storage-blob-go v0.15.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -176,6 +184,7 @@ require (
 	github.com/danieljoos/wincred v1.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/docker/cli v23.0.1+incompatible // indirect
 	github.com/docker/docker v23.0.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
@@ -191,8 +200,8 @@ require (
 	github.com/go-faster/errors v0.6.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
-	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
@@ -207,12 +216,11 @@ require (
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.13.0 // indirect
-	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
@@ -221,7 +229,6 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kr/fs v0.1.0 // indirect
-	github.com/kr/pretty v0.3.0 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -282,6 +289,10 @@ require (
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
+	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
+	gopkg.in/jcmturner/gokrb5.v6 v6.1.1 // indirect
+	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect