consent: Add foreign key migrations

See ory#1131

Signed-off-by: arekkas <[email protected]>

Makefile

@@ -1,28 +1,64 @@
+SHELL=/bin/bash -o pipefail
+
+.PHONY: test
+test:
+		docker kill hydra_test_database_mysql || true
+		docker kill hydra_test_database_postgres || true
+		docker rm -f hydra_test_database_mysql || true
+		docker rm -f hydra_test_database_postgres || true
+		docker run --rm --name hydra_test_database_mysql -p 3444:3306 -e MYSQL_ROOT_PASSWORD=secret -d mysql:5.7
+		docker run --rm --name hydra_test_database_postgres -p 3445:5432 -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=hydra -d postgres:9.6
+		make gen-sql
+		TEST_DATABASE_MYSQL='root:secret@(127.0.0.1:3444)/mysql?parseTime=true' \
+			TEST_DATABASE_POSTGRESQL='postgres://postgres:[email protected]:3445/hydra?sslmode=disable' \
+			go-acc ./... -- -failfast
+		docker rm -f hydra_test_database_mysql
+		docker rm -f hydra_test_database_postgres
+
+.PHONY: test-resetdb
+test-resetdb:
+		docker kill hydra_test_database_mysql || true
+		docker kill hydra_test_database_postgres || true
+		docker rm -f hydra_test_database_mysql || true
+		docker rm -f hydra_test_database_postgres || true
+		docker run --rm --name hydra_test_database_mysql -p 3444:3306 -e MYSQL_ROOT_PASSWORD=secret -d mysql:5.7
+		docker run --rm --name hydra_test_database_postgres -p 3445:5432 -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=hydra -d postgres:9.6
+
+.PHONY: test-short
+test-short:
+		go test -failfast -short ./...
+
+.PHONY: init
 init:
 		go get -u \
 			github.com/ory/x/tools/listx \
 			github.com/sqs/goreturns \
+			github.com/ory/go-acc \
 			github.com/golang/mock/mockgen \
 			github.com/go-swagger/go-swagger/cmd/swagger \
 			github.com/go-bindata/go-bindata/... \
 			golang.org/x/tools/cmd/goimports \
 			github.com/gobuffalo/packr/packr
 
+.PHONY: format
 format:
 		goreturns -w -local github.com/ory $$(listx .)
-		# goimports -w -local github.com/ory $$(listx .)
 
+.PHONY: gen-mocks
 gen-mocks:
 		mockgen -package oauth2_test -destination oauth2/oauth2_provider_mock_test.go github.com/ory/fosite OAuth2Provider
 
+.PHONY: gen-sql
 gen-sql:
 		cd client; go-bindata -o sql_migration_files.go -pkg client ./migrations/sql/shared ./migrations/sql/mysql ./migrations/sql/postgres ./migrations/sql/tests
 		cd consent; go-bindata -o sql_migration_files.go -pkg consent ./migrations/sql/shared ./migrations/sql/mysql ./migrations/sql/postgres ./migrations/sql/tests
 		cd jwk; go-bindata -o sql_migration_files.go -pkg jwk ./migrations/sql/shared ./migrations/sql/mysql ./migrations/sql/postgres ./migrations/sql/tests
 		cd oauth2; go-bindata -o sql_migration_files.go -pkg oauth2 ./migrations/sql/shared ./migrations/sql/mysql ./migrations/sql/postgres ./migrations/sql/tests
 
+.PHONY: gen
 gen: gen-mocks gen-sql gen-sdk
 
+.PHONY: gen-sdk
 gen-sdk:
 		swagger generate spec -m -o ./docs/api.swagger.json
 		swagger validate ./docs/api.swagger.json

README.md

@@ -273,8 +273,12 @@ Developing with ORY Hydra is as easy as:
 ```
 go get -d -u github.com/ory/hydra
 cd $GOPATH/src/github.com/ory/hydra
+make init
 export GO111MODULE=on
-go test ./...
+## With database
+make test
+## Without database
+make test-short
 ```
 
 Then run it with in-memory database:

UPGRADE.md

@@ -103,8 +103,85 @@ before finalizing the upgrade process.
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
+## Hassle-free upgrades
+
+Do you want the latest features and patches without work and hassle? Are you looking for a reliable, scalable, and
+secure deployment with zero effort? We can run it for you! If you're interested,
+[contact us now](mailto:[email protected])!
+
 ## 1.0.0-rc.1
 
+This release ships with major scalability and reliability improvements and resolves several bugs.
+
+### Schema Changes
+
+Please read all paragraphs of this section with the utmost care, before executing `hydra migrate sql`. Do
+not take this change lightly and create a backup of the database before you begin. To be sure, copy the database
+and do a dry-run locally.
+
+#### Foreign Keys
+
+In order to keep data consistent across tables, several foreign key constraints have been added between consent, oauth2, client tables.
+If you are running a large database take enough time to run this migration - it might take a while depending on the
+amount of data and the database version and driver. Before executing this migration, you should *manually* check and remove
+inconsistent data.
+
+##### Removing inconsistent login & consent data
+
+This migration automatically removes inconsistent login & consent data. Possible impacts are:
+
+1. Users that set `remember` to true during login have to re-authenticate.
+2. Users that set `remember` to true during consent have to re-authorize requested OAuth 2.0 Scope.
+3. Data associated with OAuth 2.0 Clients that have been removed will be deleted.
+
+That is achieved by running the following queries. Make sure you understand what these queries do and what impact
+they may have on your system before executing `hydra migrate sql`:
+
+```sql
+DELETE FROM hydra_oauth2_consent_request_handled WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_oauth2_consent_request WHERE hydra_oauth2_consent_request_handled.challenge = hydra_oauth2_consent_request.challenge
+);
+DELETE FROM hydra_oauth2_authentication_request_handled WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_oauth2_consent_request WHERE hydra_oauth2_authentication_request_handled.challenge = hydra_oauth2_consent_request.challenge
+);
+
+DELETE FROM hydra_oauth2_consent_request WHERE login_challenge='';
+
+DELETE FROM hydra_oauth2_authentication_request WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_client WHERE hydra_oauth2_authentication_request.client_id = hydra_client.id
+);
+DELETE FROM hydra_oauth2_authentication_request WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_oauth2_authentication_session WHERE hydra_oauth2_authentication_request.login_session_id = hydra_oauth2_authentication_session.id
+);
+
+DELETE FROM hydra_oauth2_consent_request WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_client WHERE hydra_oauth2_consent_request.client_id = hydra_client.id
+);
+DELETE FROM hydra_oauth2_consent_request WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_oauth2_authentication_session WHERE hydra_oauth2_consent_request.login_session_id = hydra_oauth2_authentication_session.id
+);
+DELETE FROM hydra_oauth2_consent_request WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_oauth2_authentication_request WHERE hydra_oauth2_consent_request.login_challenge = hydra_oauth2_authentication_request.challenge
+);
+
+DELETE FROM hydra_oauth2_obfuscated_authentication_session WHERE NOT EXISTS (
+  SELECT 1 FROM hydra_client WHERE hydra_oauth2_obfuscated_authentication_session.client_id = hydra_client.id
+);
+```
+
+Be aware that some queries might cascade and remove other data to. One such example is checking `hydra_oauth2_consent_request`
+for rows that have no associated `login_challenge`. If such a row is removed, the associated `hydra_oauth2_consent_request_handled`
+is removed as well.
+
+
+#### Indices
+
+In order to [resolve table locking](https://github.com/ory/hydra/issues/1067) during the refresh token flow, the following indices were added:
+- Unique index on the `request_id` column in the `hydra_oauth2_access` & `hydra_oauth2_refresh` tables
+
+In order to [resolve table locking](https://github.com/ory/hydra/issues/1067) when flushing expired tokens, the following index was added:
+- Index on the `requested_at` column in the `hydra_oauth2_access` table
+
 ### Non-breaking Changes
 
 #### Access Token Audience
@@ -121,17 +198,6 @@ at the client, the flow will fail and no refresh token will be granted.
 
 You can now set the login and consent flow timeout using environment variable `LOGIN_CONSENT_REQUEST_LIFESPAN`.
 
-#### Schema Changes
-
-This patch introduces database schema changes. Before you apply it, you must run `hydra migrate sql` against
-your database.
-
-In order to [resolve table locking](https://github.com/ory/hydra/issues/1067) during the refresh token flow, the following indices were added:
-- Unique index on the `request_id` column in the `hydra_oauth2_access` & `hydra_oauth2_refresh` tables
-
-In order to [resolve table locking](https://github.com/ory/hydra/issues/1067) when flushing expired tokens, the following index was added:
-- Index on the `requested_at` column in the `hydra_oauth2_access` table
-
 ### Breaking Changes
 
 #### Refresh Token Expiry