Transparent decryption with metadata replacing (cossacklabs#515)

* extend client session with map interface to store session related data

* save session in ctx

* save query encryption settings to client session

* make public function of encoding binary data to hex
wrap encoding functions with wrapper that do nothing with valid strings

* extend GetData method for BoundValue with error returning to handle possible errors

* save client session in ctx after accepting connection
user struct instead of interface for QueryDataItem to avoid extra nil checks with reflection

* remove redundant interface method with per column subscribing that
weren't used by any component and just produce unused code and complicate
reading

* extend PgSQLDataEncoderProcessor, make responsible for all data encoding/decoding
operations over interesting data for decryption/detokenization received
from database

* update encoding encrypted data in PostgresqlDBDataCoder
extend encryptor, now find ColumnEncryptionSettings for queries from Parse
Packet with placeholders and save it in ClientSession to use it in
ParameterDescription packet followed by Parse packet (and also for
Bind packet but there we already have query to re-parse it)

* parse ParameterDescription + RowDescription packets,
update OID values according to ColumnEncryptionSetting
change value encoding flow for encrypted integers
Remove encoding/decoding logic from Column struct, use only
from PostgresqlEncoderDecoder and BoundValue encodings

* refactor ColumnData:
- don't use DecodedData struct
- store raw data on protocol handler level
- encode/decode data in OnColumn handler on higher level

* log keystore's folder used on startup

* use logger related to context instead of global in handleBindPacket function

* store data about used placeholders and related ColumnEncryptionSetting in SQL queries
to encrypt bound values

* handle text format for binds too
fix parameter description updates

* validate token type in encryption_config

* fix saving placeholder's data and add unit tests

* move pgsql data encoder from pseudonymization to decryptor/postgresql package

* convert comparable data to bytes due to receiving as bytes from db drivers

* use separate setting field for data type tampering

* separate PgDataEncodeDecodeProcessor into two separate processors
fix unit/integration tests

* encapsulate long logical check into separate function into
common package decryptor/base

.circleci/check_golint.sh

@@ -7,7 +7,7 @@
 # and skip issues about `_` in packages' names since we won't rename packages just to make this issue disappear
 result=$(golint ./... | grep -v "\.pb\.go\|don't use an underscore in package name" | tee /dev/stderr | wc -l)
 
-if [[ $result -gt 6 ]]; then
+if [[ $result -gt 5 ]]; then
   # too many golint issues
   echo "Too many golint issues: $result"
   exit 1;

.circleci/check_gotest.sh

@@ -14,6 +14,15 @@ if [ -z "$GO_VERSIONS" ]; then
     GO_VERSIONS="$(readlink $GOROOT)"
 fi
 
+# for local run
+if [ -z "$GO_VERSIONS" ]; then
+  echo 'Run tests with local golang executable'
+  go test -tags="${TEST_BUILD_TAGS}" ${TEST_EXTRA_BUILD_FLAGS} ./...;
+  status="$?"
+  exit $status
+fi
+
+# for circleci run
 for go_version in $GO_VERSIONS; do
     export GOROOT="/usr/local/lib/go/$go_version"
 

CHANGELOG_DEV.md

@@ -1,3 +1,9 @@
+## 0.93.0 - 2022-03-28
+- Transparent decryption with replacing type's metadata
+- Extend `encryptor_config` with new settings: `data_type=[int32|int64|str|bytes]` and `default_data_value: <SQL int literal | string | base64 string>`
+- Support values in text format from Postgresql's binary protocol
+- Refactored internals of data encoding/decoding, protocol processing, saving session related data
+
 ## 0.93.0 - 2022-03-23
 - Remove autogeneration of poison keys on the first access (but keep in poisonrecordmaker).
 - Add warning on enabled poison detection if keys are not generated.

Makefile

@@ -192,8 +192,19 @@ build_protobuf:
 	@protoc --go_out=`pwd` --go-grpc_out=`pwd` \
 		--go_opt=module=github.com/cossacklabs/acra \
 		--go-grpc_opt=module=github.com/cossacklabs/acra \
-		-Icmd/acra-translator/grpc_api \
-		cmd/acra-translator/grpc_api/*.proto
+		-Ipseudonymization/common \
+		pseudonymization/common/*.proto
+	@protoc --go_out=`pwd` --go-grpc_out=`pwd` \
+    		--go_opt=module=github.com/cossacklabs/acra \
+    		--go-grpc_opt=module=github.com/cossacklabs/acra \
+    		-Icmd/acra-translator/grpc_api \
+    		cmd/acra-translator/grpc_api/*.proto
+	@protoc --go_out=`pwd` --go-grpc_out=`pwd` \
+    		--go_opt=module=github.com/cossacklabs/acra \
+    		--go-grpc_opt=module=github.com/cossacklabs/acra \
+    		-Iencryptor/config/common \
+    		encryptor/config/common/*.proto
+
 	@python3 -m grpc_tools.protoc -Icmd/acra-translator/grpc_api --proto_path=. --python_out=tests/ --grpc_python_out=tests/ cmd/acra-translator/grpc_api/*.proto
 
 ## Build the application in the subdirectory (default)

benchmarks/common/db.go

@@ -17,6 +17,7 @@ package common
 
 import (
 	"database/sql"
+	// import driver for connect function
 	_ "github.com/jackc/pgx/v4/stdlib"
 	"github.com/sirupsen/logrus"
 	"os"

cmd/acra-server/acra-server.go

@@ -318,7 +318,7 @@ func realMain() error {
 	}
 
 	serverConfig.SetKeyStore(keyStore)
-	log.Infof("Keystore init OK")
+	log.WithField("path", *keysDir).Infof("Keystore init OK")
 
 	if err := crypto.InitRegistry(keyStore); err != nil {
 		log.WithError(err).Errorln("Can't initialize crypto registry")

cmd/acra-server/common/client_session.go

@@ -36,6 +36,7 @@ type ClientSession struct {
 	logger         *log.Entry
 	statements     base.PreparedStatementRegistry
 	protocolState  interface{}
+	data           map[string]interface{}
 }
 
 var sessionCounter uint32
@@ -47,8 +48,35 @@ func NewClientSession(ctx context.Context, config *Config, connection net.Conn)
 	sessionID := atomic.AddUint32(&sessionCounter, 1)
 	logger := logging.GetLoggerFromContext(ctx)
 	logger = logger.WithField("session_id", sessionID)
+	session := &ClientSession{connection: connection, config: config, ctx: ctx, logger: logger,
+		data: make(map[string]interface{}, 8)}
 	ctx = logging.SetLoggerToContext(ctx, logger)
-	return &ClientSession{connection: connection, config: config, ctx: ctx, logger: logger}, nil
+	ctx = base.SetClientSessionToContext(ctx, session)
+	session.ctx = ctx
+	return session, nil
+
+}
+
+// SetData save session related data by key
+func (clientSession *ClientSession) SetData(key string, data interface{}) {
+	clientSession.data[key] = data
+}
+
+// GetData return session related data by key and true otherwise nil, false
+func (clientSession *ClientSession) GetData(key string) (interface{}, bool) {
+	value, ok := clientSession.data[key]
+	return value, ok
+}
+
+// DeleteData delete session related data by key
+func (clientSession *ClientSession) DeleteData(key string) {
+	delete(clientSession.data, key)
+}
+
+// HasData return true if session has data by key
+func (clientSession *ClientSession) HasData(key string) bool {
+	_, ok := clientSession.data[key]
+	return ok
 }
 
 // Logger returns session's logger.

cmd/acra-server/common/client_session_test.go

@@ -0,0 +1,73 @@
+package common
+
+import (
+	"context"
+	"reflect"
+	"testing"
+)
+
+func TestClientSession_Data(t *testing.T) {
+	session, err := NewClientSession(context.TODO(), nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	type testcase struct {
+		key  string
+		data interface{}
+	}
+	testcases := []testcase{
+		{`binary key`, []byte(`binary data`)},
+		{`string key`, `string value`},
+		{`int key`, 123},
+		{`struct key`, testcase{`123`, `123`}},
+	}
+	overwriteValue := `some value that will overwrite existing value`
+	for _, tcase := range testcases {
+		if session.HasData(tcase.key) {
+			t.Fatal("session should not have value of not used key")
+		}
+		value, ok := session.GetData(tcase.key)
+		if ok {
+			t.Fatal("session should not have value of not used key")
+		}
+		if value != nil {
+			t.Fatal("session should return nil for not existing keys")
+		}
+		session.SetData(tcase.key, tcase.data)
+		if !session.HasData(tcase.key) {
+			t.Fatal("session hasn't value of existing key")
+		}
+		value, ok = session.GetData(tcase.key)
+		if !ok {
+			t.Fatal("session hasn't value of of existing key")
+		}
+		if !reflect.DeepEqual(tcase.data, value) {
+			t.Fatal("session returned another value")
+		}
+
+		// overwrite value and check that it successfully overwritten
+		session.SetData(tcase.key, overwriteValue)
+		if !session.HasData(tcase.key) {
+			t.Fatal("session hasn't value of existing key")
+		}
+		value, ok = session.GetData(tcase.key)
+		if !ok {
+			t.Fatal("session hasn't value of of existing key")
+		}
+		if !reflect.DeepEqual(overwriteValue, value) {
+			t.Fatal("session returned another value")
+		}
+
+		session.DeleteData(tcase.key)
+		if session.HasData(tcase.key) {
+			t.Fatal("session should not have value of not used key")
+		}
+		value, ok = session.GetData(tcase.key)
+		if ok {
+			t.Fatal("session should not have value of not used key")
+		}
+		if value != nil {
+			t.Fatal("session should return nil for not existing keys")
+		}
+	}
+}

cmd/acra-server/common/listener.go

@@ -184,18 +184,18 @@ func (server *SServer) handleClientSession(clientID []byte, clientSession *Clien
 	accessContext := base.NewAccessContext(base.WithClientID(clientID), base.WithZoneMode(server.config.GetWithZone()))
 	// subscribe on clientID changes after switching connection to TLS and using ClientID from TLS certificates
 	proxy.AddClientIDObserver(accessContext)
-
+	clientSession.ctx = base.SetAccessContextToContext(clientSession.ctx, accessContext)
 	server.backgroundWorkersSync.Add(1)
 	go func() {
 		defer server.backgroundWorkersSync.Done()
 		defer recoverConnection(sessionLogger.WithField("function", "ProxyClientConnection"), sessionCloseToCloser(clientSession.Close))
-		proxy.ProxyClientConnection(base.SetAccessContextToContext(clientSession.ctx, accessContext), proxyErrCh)
+		proxy.ProxyClientConnection(clientSession.ctx, proxyErrCh)
 	}()
 	server.backgroundWorkersSync.Add(1)
 	go func() {
 		defer server.backgroundWorkersSync.Done()
 		defer recoverConnection(sessionLogger.WithField("function", "ProxyDatabaseConnection"), sessionCloseToCloser(clientSession.Close))
-		proxy.ProxyDatabaseConnection(base.SetAccessContextToContext(clientSession.ctx, accessContext), proxyErrCh)
+		proxy.ProxyDatabaseConnection(clientSession.ctx, proxyErrCh)
 	}()
 
 	proxyErr := <-proxyErrCh

cmd/acra-translator/grpc_api/Readme.md

@@ -1,10 +1,9 @@
 # Install grpc dependencies
 ```
-# from https://github.com/grpc/grpc-go
-go get -u github.com/golang/protobuf/{proto,protoc-gen-go}
-go get -u google.golang.org/grpc
+# from https://developers.google.com/protocol-buffers/docs/gotutorial
+go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
 ```
 To recompile proto file run from root of acra repository:
 ```
-protoc --go_out=plugins=grpc:. cmd/acra-translator/grpc_api/api.proto
+make build_protobuf
 ```

crypto/envelope_detector.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"github.com/cossacklabs/acra/acrablock"
 	"github.com/cossacklabs/acra/acrastruct"
+	"github.com/cossacklabs/acra/decryptor/base"
 	"github.com/cossacklabs/acra/utils"
 	"github.com/sirupsen/logrus"
 )
@@ -60,6 +61,7 @@ func (recognizer *EnvelopeDetector) OnColumn(ctx context.Context, inBuffer []byt
 		return ctx, inBuffer, nil
 	}
 	outBuffer := make([]byte, 0, len(inBuffer))
+	changed := false
 	// inline mode
 	inIndex := 0
 	for {
@@ -96,6 +98,7 @@ func (recognizer *EnvelopeDetector) OnColumn(ctx context.Context, inBuffer []byt
 			if !bytes.Equal(processedData, container) {
 				outBuffer = append(outBuffer, processedData...)
 				inIndex += n
+				changed = true
 				break
 			}
 
@@ -109,6 +112,9 @@ func (recognizer *EnvelopeDetector) OnColumn(ctx context.Context, inBuffer []byt
 	}
 	// copy left bytes
 	outBuffer = append(outBuffer, inBuffer[inIndex:]...)
+	if changed {
+		return base.MarkDecryptedContext(ctx), outBuffer, nil
+	}
 	return ctx, outBuffer, nil
 }
 
@@ -194,6 +200,9 @@ func (wrapper *OldContainerDetectorWrapper) OnColumn(ctx context.Context, inBuff
 	if err != nil {
 		return ctx, inBuffer, err
 	}
+	if !bytes.Equal(inBuffer, outBuffer) {
+		return base.MarkDecryptedContext(ctx), outBuffer, nil
+	}
 
 	return ctx, outBuffer, nil
 }

crypto/envelope_detector_test.go

@@ -183,10 +183,13 @@ func TestOldContainerDetectorWrapper(t *testing.T) {
 				},
 			}
 			for _, tcase := range testCases {
-				_, outBuffer, err := containerDetector.OnColumn(base.SetAccessContextToContext(context.Background(), accessContext), tcase.input)
+				ctx, outBuffer, err := containerDetector.OnColumn(base.SetAccessContextToContext(context.Background(), accessContext), tcase.input)
 				if err != nil {
 					t.Fatal("OnColumn error ", err)
 				}
+				if !base.IsDecryptedFromContext(ctx) {
+					t.Fatal("Expects decrypted data")
+				}
 
 				if !bytes.Equal(outBuffer, tcase.expected) {
 					t.Fatal("outBuffer is not equals to expected", err)

decryptor/base/clientSession.go

@@ -0,0 +1,40 @@
+package base
+
+import (
+	"context"
+	"net"
+)
+
+// ClientSession is a connection between the client and the database, mediated by AcraServer.
+type ClientSession interface {
+	Context() context.Context
+	ClientConnection() net.Conn
+	DatabaseConnection() net.Conn
+
+	PreparedStatementRegistry() PreparedStatementRegistry
+	SetPreparedStatementRegistry(registry PreparedStatementRegistry)
+
+	ProtocolState() interface{}
+	SetProtocolState(state interface{})
+	GetData(string) (interface{}, bool)
+	SetData(string, interface{})
+	DeleteData(string)
+	HasData(string) bool
+}
+
+type sessionContextKey struct{}
+
+// SetClientSessionToContext return context with saved ClientSession
+func SetClientSessionToContext(ctx context.Context, session ClientSession) context.Context {
+	return context.WithValue(ctx, sessionContextKey{}, session)
+}
+
+// ClientSessionFromContext return saved ClientSession from context or nil
+func ClientSessionFromContext(ctx context.Context) ClientSession {
+	value := ctx.Value(sessionContextKey{})
+	session, ok := value.(ClientSession)
+	if ok {
+		return session
+	}
+	return nil
+}