Add ExtraProgramPages app call field support (algorand#545)

* Add ExtraProgramPages app call field support

* Add TotalExtraAppPages and TotalAppSchema calculated fields to account response

* Upgrade Go SDK to v1.9.1

accounting/accounting.go

@@ -394,6 +394,7 @@ func (accounting *State) AddTransaction(txnr *idb.TxnRow) (err error) {
 				ClearStateProgram: stxn.Txn.ClearStateProgram,
 				LocalStateSchema:  stxn.Txn.LocalStateSchema,
 				GlobalStateSchema: stxn.Txn.GlobalStateSchema,
+				ExtraProgramPages: stxn.Txn.ExtraProgramPages,
 			}
 			if stxn.Txn.ApplicationID == 0 {
 				// app creation

api/converter_utils.go

@@ -382,7 +382,8 @@ func txnRowToTransaction(row idb.TxnRow) (generated.Transaction, error) {
 				NumByteSlice: stxn.Txn.LocalStateSchema.NumByteSlice,
 				NumUint:      stxn.Txn.LocalStateSchema.NumUint,
 			},
-			OnCompletion: onCompletionToTransactionOnCompletion(stxn.Txn.OnCompletion),
+			OnCompletion:      onCompletionToTransactionOnCompletion(stxn.Txn.OnCompletion),
+			ExtraProgramPages: uint64PtrOrNil(uint64(stxn.Txn.ExtraProgramPages)),
 		}
 
 		application = &a

api/generate.sh

@@ -14,5 +14,5 @@ echo "generating code."
 oapi-codegen -package generated -type-mappings integer=uint64 -generate types -o generated/v2/types.go -exclude-tags=common indexer.oas3.yml
 oapi-codegen -package generated -type-mappings integer=uint64 -generate server,spec -o generated/v2/routes.go -exclude-tags=common indexer.oas3.yml
 
-oapi-codegen -package generated -type-mappings integer=uint64 -generate types -o generated/common/types.go -include-tags=common indexer.oas3.yml
-oapi-codegen -package generated -type-mappings integer=uint64 -generate server,spec -o generated/common/routes.go -include-tags=common indexer.oas3.yml
+oapi-codegen -package common -type-mappings integer=uint64 -generate types -o generated/common/types.go -include-tags=common indexer.oas3.yml
+oapi-codegen -package common -type-mappings integer=uint64 -generate server,spec -o generated/common/routes.go -include-tags=common indexer.oas3.yml

api/indexer.oas2.json

@@ -656,6 +656,10 @@
           "description": "\\[tsch\\] stores the sum of all of the local schemas and global schemas in this account.\n\nNote: the raw account uses `StateSchema` for this type.",
           "$ref": "#/definitions/ApplicationStateSchema"
         },
+        "apps-total-extra-pages": {
+          "description": "\\[teap\\] the sum of all extra application program pages for this account.",
+          "type": "integer"
+        },
         "assets": {
           "description": "\\[asset\\] assets held by this account.\n\nNote the raw object uses `map[int] -\u003e AssetHolding` for this type.",
           "type": "array",
@@ -934,6 +938,10 @@
         "global-state": {
           "description": "[\\gs\\] global schema",
           "$ref": "#/definitions/TealKeyValueStore"
+        },
+        "extra-program-pages": {
+          "description": "\\[epp\\] the amount of extra program pages available to this app.",
+          "type": "integer"
         }
       }
     },
@@ -1625,6 +1633,10 @@
           "type": "string",
           "format": "byte",
           "x-algorand-format": "TEALProgram"
+        },
+        "extra-program-pages": {
+          "description": "\\[epp\\] specifies the additional app program len requested in pages.",
+          "type": "integer"
         }
       }
     },

api/indexer.oas3.yml

@@ -503,6 +503,10 @@
             },
             "type": "array"
           },
+          "apps-total-extra-pages": {
+            "description": "\\[teap\\] the sum of all extra application program pages for this account.",
+            "type": "integer"
+          },
           "apps-total-schema": {
             "$ref": "#/components/schemas/ApplicationStateSchema"
           },
@@ -730,6 +734,10 @@
             "type": "string",
             "x-algorand-format": "Address"
           },
+          "extra-program-pages": {
+            "description": "\\[epp\\] the amount of extra program pages available to this app.",
+            "type": "integer"
+          },
           "global-state": {
             "$ref": "#/components/schemas/TealKeyValueStore"
           },
@@ -1283,7 +1291,7 @@
             "$ref": "#/components/schemas/TransactionAssetTransfer"
           },
           "auth-addr": {
-            "description": "\\[sgnr\\] The address used to sign the transaction. This is used for rekeyed accounts to indicate that the sender address did not sign the transaction.",
+            "description": "\\[sgnr\\] this is included with signed transactions when the signing address does not equal the sender. The backend can use this to ensure that auth addr is equal to the accounts auth addr.",
             "type": "string",
             "x-algorand-format": "Address"
           },
@@ -1456,6 +1464,10 @@
             "type": "string",
             "x-algorand-format": "TEALProgram"
           },
+          "extra-program-pages": {
+            "description": "\\[epp\\] specifies the additional app program len requested in pages.",
+            "type": "integer"
+          },
           "foreign-apps": {
             "description": "\\[apfa\\] Lists the applications in addition to the application-id whose global states may be accessed by this application's approval-program and clear-state-program. The access is read-only.",
             "items": {

api/pointer_utils.go

@@ -44,6 +44,13 @@ func uint64Ptr(x uint64) *uint64 {
 	return &x
 }
 
+func uint64PtrOrNil(x uint64) *uint64 {
+	if x == 0 {
+		return nil
+	}
+	return &x
+}
+
 func bytePtr(x []byte) *[]byte {
 	if len(x) == 0 {
 		return nil

cmd/validator/struct_processor.go

@@ -70,12 +70,12 @@ func equals(indexer, algod generated.Account) (differences []string) {
 	if algod.AmountWithoutPendingRewards != indexer.AmountWithoutPendingRewards {
 		differences = append(differences, "amount-without-pending-rewards")
 	}
-	/*
-		// Indexer doesn't support this yet.
-		if !appSchemaComparePtrEqual(algod.AppsTotalSchema, indexer.AppsTotalSchema) {
-			differences = append(differences, "apps-total-schema")
-		}
-	*/
+	if !appSchemaComparePtrEqual(algod.AppsTotalSchema, indexer.AppsTotalSchema) {
+		differences = append(differences, "apps-total-schema")
+	}
+	if !uint64PtrEqual(algod.AppsTotalExtraPages, indexer.AppsTotalExtraPages) {
+		differences = append(differences, "apps-total-extra-pages")
+	}
 	if !stringPtrEqual(algod.AuthAddr, indexer.AuthAddr) {
 		// Indexer doesn't remove the auth addr when it is removed.
 		if indexer.AuthAddr == nil || *indexer.AuthAddr != indexer.Address {
@@ -247,6 +247,9 @@ func equals(indexer, algod generated.Account) (differences []string) {
 			if !stateSchemePtrEqual(algodCreatedApp.Params.GlobalStateSchema, indexerCreatedApp.Params.GlobalStateSchema) {
 				differences = append(differences, fmt.Sprintf("created-app global-state-schema %d", algodCreatedApp.Id))
 			}
+			if !uint64PtrEqual(algodCreatedApp.Params.ExtraProgramPages, indexerCreatedApp.Params.ExtraProgramPages) {
+				differences = append(differences, fmt.Sprintf("created-app extra-pages %d", algodCreatedApp.Id))
+			}
 		}
 	}
 

go.mod

@@ -3,7 +3,7 @@ module github.com/algorand/indexer
 go 1.13
 
 require (
-	github.com/algorand/go-algorand-sdk v1.7.0
+	github.com/algorand/go-algorand-sdk v1.9.1
 	github.com/algorand/go-codec/codec v1.1.7
 	github.com/algorand/oapi-codegen v1.3.5-algorand5
 	github.com/getkin/kin-openapi v0.19.0

go.sum

@@ -62,6 +62,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/algorand/go-algorand-sdk v1.7.0 h1:eRg9UQy7FndRE5nTgniDpbyjTvP2FUKX4/0e8NdXn/E=
 github.com/algorand/go-algorand-sdk v1.7.0/go.mod h1:U12d8fTN/CyKPR1HObrt51ITxb6OgXxpGCH743Ds2GQ=
+github.com/algorand/go-algorand-sdk v1.9.0 h1:3xhFaScyMVy6D/XAWAuTnboNsnkgl9EqCZqgAnaOLm4=
+github.com/algorand/go-algorand-sdk v1.9.0/go.mod h1:U12d8fTN/CyKPR1HObrt51ITxb6OgXxpGCH743Ds2GQ=
+github.com/algorand/go-algorand-sdk v1.9.1 h1:v2UaVXeMOZxvWoNJp6+MUAjm8Gif+ultcxg1RBrj45s=
+github.com/algorand/go-algorand-sdk v1.9.1/go.mod h1:U12d8fTN/CyKPR1HObrt51ITxb6OgXxpGCH743Ds2GQ=
 github.com/algorand/go-codec v1.1.7 h1:6nvCh2nfgnfkaoVHKQyk2wxyl2GQBAlI7IkbqbB/e4s=
 github.com/algorand/go-codec v1.1.7/go.mod h1:pVLQYhIVCsx9D3iy4W4Qqi0SKhx6IVhMwOvj/agFL4g=
 github.com/algorand/go-codec/codec v1.1.7 h1:EFOyWf5duxbh2ru+AW1YDgmZ+MRVgqklELSqTArgp3M=

idb/idb.go

@@ -405,6 +405,7 @@ type AppDelta struct {
 	ClearStateProgram []byte                `codec:"clearp"`
 	LocalStateSchema  sdk_types.StateSchema `codec:"lsch"`
 	GlobalStateSchema sdk_types.StateSchema `codec:"gsch"`
+	ExtraProgramPages uint32                `codec:"epp"`
 }
 
 // String is part of the Stringer interface.
@@ -434,6 +435,9 @@ func (ad AppDelta) String() string {
 	if ad.LocalStateSchema.NumByteSlice != 0 || ad.LocalStateSchema.NumUint != 0 {
 		parts = append(parts, fmt.Sprintf("lss(b=%d, i=%d)", ad.LocalStateSchema.NumByteSlice, ad.LocalStateSchema.NumUint))
 	}
+	if ad.ExtraProgramPages != 0 {
+		parts = append(parts, fmt.Sprintf("epp=%d", ad.ExtraProgramPages))
+	}
 
 	return strings.Join(parts, " ")
 }
@@ -452,6 +456,7 @@ type AppReverseDelta struct {
 	ClearStateProgram []byte                 `codec:"clearp,omitempty"`
 	LocalStateSchema  sdk_types.StateSchema  `codec:"lsch,omitempty"`
 	GlobalStateSchema sdk_types.StateSchema  `codec:"gsch,omitempty"`
+	ExtraProgramPages uint32                 `codec:"epp,omitempty"`
 }
 
 // SetDelta adds delta values to the AppReverseDelta object.

idb/postgres/postgres.go

@@ -720,6 +720,7 @@ type AppParams struct {
 	ClearStateProgram []byte      `codec:"clearp"`
 	LocalStateSchema  StateSchema `codec:"lsch"`
 	GlobalStateSchema StateSchema `codec:"gsch"`
+	ExtraProgramPages uint32      `codec:"epp"`
 
 	GlobalState TealKeyValue `codec:"gs,allocbound=-"`
 }
@@ -1136,6 +1137,9 @@ ON CONFLICT (addr, assetid) DO UPDATE SET amount = account_asset.amount + EXCLUD
 					return fmt.Errorf("app delta apply err r=%d i=%d app=%d, %v", adelta.Round, adelta.Intra, adelta.AppIndex, err)
 				}
 			}
+			reverseDelta.ExtraProgramPages = state.ExtraProgramPages
+			state.ExtraProgramPages = adelta.ExtraProgramPages
+
 			reverseDeltas = append(reverseDeltas, []interface{}{encoding.EncodeJSON(reverseDelta), adelta.Round, adelta.Intra})
 			if adelta.OnCompletion == sdk_types.DeleteApplicationOC {
 				// clear content but leave row recording that it existed
@@ -2092,6 +2096,8 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) {
 			*account.CreatedAssets = cal
 		}
 
+		var totalSchema models.ApplicationStateSchema
+
 		if len(appParamIndexes) > 0 {
 			// apps owned by this account
 			var appIds []uint64
@@ -2136,6 +2142,7 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) {
 				break
 			}
 
+			var totalExtraPages uint64
 			aout := make([]models.Application, len(appIds))
 			outpos := 0
 			for i, appid := range appIds {
@@ -2160,13 +2167,22 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) {
 						NumUint:      apps[i].LocalStateSchema.NumUint,
 					}
 				}
+				if aout[outpos].Deleted == nil || !*aout[outpos].Deleted {
+					totalSchema.NumByteSlice += apps[i].GlobalStateSchema.NumByteSlice
+					totalSchema.NumUint += apps[i].GlobalStateSchema.NumUint
+					totalExtraPages += uint64(apps[i].ExtraProgramPages)
+				}
 
 				outpos++
 			}
 			if outpos != len(aout) {
 				aout = aout[:outpos]
 			}
 			account.CreatedApps = &aout
+
+			if totalExtraPages != 0 {
+				account.AppsTotalExtraPages = &totalExtraPages
+			}
 		}
 
 		if len(localStateAppIds) > 0 {
@@ -2222,10 +2238,19 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) {
 					NumUint:      ls[i].Schema.NumUint,
 				}
 				aout[i].KeyValue = ls[i].KeyValue.toModel()
+				if aout[i].Deleted == nil || !*aout[i].Deleted {
+					totalSchema.NumByteSlice += ls[i].Schema.NumByteSlice
+					totalSchema.NumUint += ls[i].Schema.NumUint
+				}
+
 			}
 			account.AppsLocalState = &aout
 		}
 
+		if totalSchema != (models.ApplicationStateSchema{}) {
+			account.AppsTotalSchema = &totalSchema
+		}
+
 		// Sometimes the migration state effects what data should be returned.
 		db.processAccount(&account)
 
@@ -2876,6 +2901,12 @@ func (db *IndexerDb) yieldApplicationsThread(ctx context.Context, rows *sql.Rows
 			NumByteSlice: ap.LocalStateSchema.NumByteSlice,
 			NumUint:      ap.LocalStateSchema.NumUint,
 		}
+
+		if ap.ExtraProgramPages != 0 {
+			rec.Application.Params.ExtraProgramPages = new(uint64)
+			*rec.Application.Params.ExtraProgramPages = uint64(ap.ExtraProgramPages)
+		}
+
 		out <- rec
 	}
 	if err := rows.Err(); err != nil {