From 6d220c3315887c140327c9b1340fba286a2dd28f Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Thu, 12 Mar 2020 08:22:51 +1100 Subject: [PATCH] Set up CI for Python 3. (#1546) --- Pipfile | 1 - Pipfile.lock | 533 +++++++++++++------------- cloudbuild3.yaml | 54 +++ docker/ci/Dockerfile | 8 + src/local/butler/integration_tests.py | 16 +- 5 files changed, 332 insertions(+), 280 deletions(-) create mode 100644 cloudbuild3.yaml diff --git a/Pipfile b/Pipfile index 3349594337..3934e76240 100644 --- a/Pipfile +++ b/Pipfile @@ -26,7 +26,6 @@ waitress = "==1.4.3" WebTest = "==2.0.23" yapf = "==0.22.0" -[dev-packages] crcmod = "==1.7" Fabric = "==1.14.1" future = "==0.17.1" diff --git a/Pipfile.lock b/Pipfile.lock index 2d48cb8b65..e85812ea27 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5823af4820337f9fe82b96dcf181fbf4603c7ed8d2341a1b7c5c2afcd8d12191" + "sha256": "74fb16efacff002d6f9be2d56fef7da3ef3c28806bead81a237649fa31b0f3de" }, "pipfile-spec": 6, "requires": { @@ -16,279 +16,6 @@ ] }, "default": { - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "version": "==1.4.0" - }, - "astroid": { - "hashes": [ - "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", - "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" - ], - "version": "==2.3.3" - }, - "beautifulsoup4": { - "hashes": [ - "sha256:35815098d9d0bf5d3a782346a3945b000592d0e93e2538a502d0d6807130d675", - "sha256:3c9474036afda9136aac6463def733f81017bf9ef3510d25634f335b0c87f5e1", - "sha256:bb3dba571c2f7ddcaf3e70ad8b67878a85ac41889dcba75a30b8f7b0b5960b6d" - ], - "index": "pypi", - "version": "==4.5.1" - }, - "ecdsa": { - "hashes": [ - "sha256:163c80b064a763ea733870feb96f9dd9b92216cfcacd374837af18e4e8ec3d4d", - "sha256:9814e700890991abeceeb2242586024d4758c8fc18445b194a49bd62d85861db" - ], - "index": "pypi", - "version": "==0.13.3" - }, - "extras": { - "hashes": [ - "sha256:132e36de10b9c91d5d4cc620160a476e0468a88f16c9431817a6729611a81b4e", - "sha256:f689f08df47e2decf76aa6208c081306e7bd472630eb1ec8a875c67de2366e87" - ], - "index": "pypi", - "version": "==1.0.0" - }, - "fixtures": { - "hashes": [ - "sha256:2a551b0421101de112d9497fb5f6fd25e5019391c0fbec9bad591ecae981420d", - "sha256:fcf0d60234f1544da717a9738325812de1f42c2fa085e2d9252d8fff5712b2ef" - ], - "index": "pypi", - "version": "==3.0.0" - }, - "funcsigs": { - "hashes": [ - "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", - "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" - ], - "index": "pypi", - "version": "==1.0.2" - }, - "isort": { - "hashes": [ - "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", - "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" - ], - "version": "==4.3.21" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", - "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", - "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", - "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", - "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", - "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", - "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", - "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", - "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", - "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", - "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", - "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", - "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", - "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", - "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", - "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", - "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", - "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", - "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", - "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", - "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" - ], - "version": "==1.4.3" - }, - "linecache2": { - "hashes": [ - "sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c", - "sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef" - ], - "index": "pypi", - "version": "==1.0.0" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "mock": { - "hashes": [ - "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", - "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" - ], - "index": "pypi", - "version": "==2.0.0" - }, - "mox3": { - "hashes": [ - "sha256:01faf6ae7cebda8e570b62795e063b48908d1c9aaa857da9c1b5997921c0c83a", - "sha256:fbe9f71ffcd3ce2f46770effe5d429d62b674560a57c76f347877179056735e5" - ], - "index": "pypi", - "version": "==0.18.0" - }, - "parameterized": { - "hashes": [ - "sha256:caf58e717097735de0d7e15386a46ffa5ce25bb6a13a43716a8854a8d34841e2", - "sha256:cf5fa4f295dfb823cebdb27a00566113f2fbb71c7d5ca7b7a1019fd20c8a0811" - ], - "index": "pypi", - "version": "==0.6.1" - }, - "pbr": { - "hashes": [ - "sha256:186428c270309e6fdfe2d5ab0949ab21ae5f7dea831eab96701b86bd666af39c", - "sha256:f5cf7265a80636ecff66806d13494cbf9d77a3758a65fd8b4d4d4bee81b0c375" - ], - "index": "pypi", - "version": "==1.10.0" - }, - "pyfakefs": { - "hashes": [ - "sha256:226e8f141e40c19d10693f8df4968407eb47100705796640ee0dc5990bd6f109", - "sha256:8312f143b37b014cdd4641ac1f4eac0c47bb881774a3d0be125997133e609a6b" - ], - "index": "pypi", - "version": "==3.4.3" - }, - "pylint": { - "hashes": [ - "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", - "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" - ], - "index": "pypi", - "version": "==2.4.4" - }, - "python-mimeparse": { - "hashes": [ - "sha256:bef134a59598cc6aa598f84553162aa7a0c01f3f431588225bb9a208964b1827" - ], - "index": "pypi", - "version": "==1.5.2" - }, - "pyyaml": { - "hashes": [ - "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", - "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", - "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", - "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", - "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", - "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", - "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", - "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", - "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", - "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", - "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" - ], - "index": "pypi", - "version": "==5.1" - }, - "six": { - "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" - ], - "index": "pypi", - "version": "==1.12.0" - }, - "testtools": { - "hashes": [ - "sha256:80f606607a6e4ce4d0e24e5b786562aa42c581906f3c070607a4265f3da65810", - "sha256:9b21a293cd33853956b1d3834c294d77a6ad0ab0eb1c077f858be433f0f225bb" - ], - "index": "pypi", - "version": "==2.2.0" - }, - "traceback2": { - "hashes": [ - "sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030", - "sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23" - ], - "index": "pypi", - "version": "==1.4.0" - }, - "typed-ast": { - "hashes": [ - "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", - "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", - "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", - "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", - "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", - "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", - "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", - "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", - "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", - "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", - "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", - "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", - "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", - "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", - "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", - "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", - "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", - "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", - "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", - "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", - "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" - ], - "markers": "implementation_name == 'cpython' and python_version < '3.8'", - "version": "==1.4.1" - }, - "unittest2": { - "hashes": [ - "sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8", - "sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579" - ], - "index": "pypi", - "version": "==1.1.0" - }, - "waitress": { - "hashes": [ - "sha256:045b3efc3d97c93362173ab1dfc159b52cfa22b46c3334ffc805dbdbf0e4309e", - "sha256:77ff3f3226931a1d7d8624c5371de07c8e90c7e5d80c5cc660d72659aaf23f38" - ], - "index": "pypi", - "version": "==1.4.3" - }, - "webob": { - "hashes": [ - "sha256:a3c89a8e9ba0aeb17382836cdb73c516d0ecf6630ec40ec28288f3ed459ce87b", - "sha256:aa3a917ed752ba3e0b242234b2a373f9c4e2a75d35291dcbe977649bd21fd108" - ], - "version": "==1.8.6" - }, - "webtest": { - "hashes": [ - "sha256:0831371d415fac39177d1fab4e180510713401a71edd9181373efe123e85cb3e", - "sha256:879da5c2310b75c74dffa23a45a7a74d09c2a0fcc7e7a128b4f92a4f64722e97" - ], - "index": "pypi", - "version": "==2.0.23" - }, - "wrapt": { - "hashes": [ - "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" - ], - "version": "==1.11.2" - }, - "yapf": { - "hashes": [ - "sha256:6567745f0b6656f9c33a73c56a393071c699e6284a70d793798ab6e3769d25ec", - "sha256:a98a6eacca64d2b920558f4a2f78150db9474de821227e60deaa29f186121c63" - ], - "index": "pypi", - "version": "==0.22.0" - } - }, - "develop": { "absl-py": { "hashes": [ "sha256:75e737d6ce7723d9ff9b7aa1ba3233c34be62ef18d5859e706b8fdc828989830" @@ -296,6 +23,13 @@ "index": "pypi", "version": "==0.9.0" }, + "argparse": { + "hashes": [ + "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", + "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" + ], + "version": "==1.4.0" + }, "astor": { "hashes": [ "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", @@ -304,6 +38,13 @@ "index": "pypi", "version": "==0.8.1" }, + "astroid": { + "hashes": [ + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + ], + "version": "==2.3.3" + }, "bcrypt": { "hashes": [ "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", @@ -327,6 +68,15 @@ ], "version": "==3.1.7" }, + "beautifulsoup4": { + "hashes": [ + "sha256:35815098d9d0bf5d3a782346a3945b000592d0e93e2538a502d0d6807130d675", + "sha256:3c9474036afda9136aac6463def733f81017bf9ef3510d25634f335b0c87f5e1", + "sha256:bb3dba571c2f7ddcaf3e70ad8b67878a85ac41889dcba75a30b8f7b0b5960b6d" + ], + "index": "pypi", + "version": "==4.5.1" + }, "boto": { "hashes": [ "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8", @@ -410,6 +160,22 @@ ], "version": "==1.4.0" }, + "ecdsa": { + "hashes": [ + "sha256:163c80b064a763ea733870feb96f9dd9b92216cfcacd374837af18e4e8ec3d4d", + "sha256:9814e700890991abeceeb2242586024d4758c8fc18445b194a49bd62d85861db" + ], + "index": "pypi", + "version": "==0.13.3" + }, + "extras": { + "hashes": [ + "sha256:132e36de10b9c91d5d4cc620160a476e0468a88f16c9431817a6729611a81b4e", + "sha256:f689f08df47e2decf76aa6208c081306e7bd472630eb1ec8a875c67de2366e87" + ], + "index": "pypi", + "version": "==1.0.0" + }, "fabric": { "hashes": [ "sha256:2bb6c6922cbdfe35884c937bfcff63c70750c18456b0707898112c5ceaab38c6", @@ -418,6 +184,22 @@ "index": "pypi", "version": "==1.14.1" }, + "fixtures": { + "hashes": [ + "sha256:2a551b0421101de112d9497fb5f6fd25e5019391c0fbec9bad591ecae981420d", + "sha256:fcf0d60234f1544da717a9738325812de1f42c2fa085e2d9252d8fff5712b2ef" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "funcsigs": { + "hashes": [ + "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", + "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" + ], + "index": "pypi", + "version": "==1.0.2" + }, "future": { "hashes": [ "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" @@ -569,6 +351,13 @@ "index": "pypi", "version": "==2.10.0" }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, "keras-applications": { "hashes": [ "sha256:5579f9a12bcde9748f4a12233925a59b93b73ae6947409ff34aa2ba258189fe5", @@ -585,6 +374,40 @@ "index": "pypi", "version": "==1.1.0" }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, + "linecache2": { + "hashes": [ + "sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c", + "sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef" + ], + "index": "pypi", + "version": "==1.0.0" + }, "markdown": { "hashes": [ "sha256:90fee683eeabe1a92e149f7ba74e5ccdc81cd397bd6c516d93a8da0ef90b6902", @@ -593,6 +416,13 @@ "index": "pypi", "version": "==3.2.1" }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, "mock": { "hashes": [ "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", @@ -601,6 +431,14 @@ "index": "pypi", "version": "==2.0.0" }, + "mox3": { + "hashes": [ + "sha256:01faf6ae7cebda8e570b62795e063b48908d1c9aaa857da9c1b5997921c0c83a", + "sha256:fbe9f71ffcd3ce2f46770effe5d429d62b674560a57c76f347877179056735e5" + ], + "index": "pypi", + "version": "==0.18.0" + }, "nodeenv": { "hashes": [ "sha256:def2a6d927bef8d17c1776edbd5bbc8b7a5f0eee159af53b9924d559fc8d3202" @@ -637,6 +475,14 @@ "index": "pypi", "version": "==1.16.4" }, + "parameterized": { + "hashes": [ + "sha256:caf58e717097735de0d7e15386a46ffa5ce25bb6a13a43716a8854a8d34841e2", + "sha256:cf5fa4f295dfb823cebdb27a00566113f2fbb71c7d5ca7b7a1019fd20c8a0811" + ], + "index": "pypi", + "version": "==0.6.1" + }, "paramiko": { "hashes": [ "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", @@ -645,6 +491,14 @@ "index": "pypi", "version": "==2.6.0" }, + "pbr": { + "hashes": [ + "sha256:186428c270309e6fdfe2d5ab0949ab21ae5f7dea831eab96701b86bd666af39c", + "sha256:f5cf7265a80636ecff66806d13494cbf9d77a3758a65fd8b4d4d4bee81b0c375" + ], + "index": "pypi", + "version": "==1.10.0" + }, "protobuf": { "hashes": [ "sha256:10394a4d03af7060fa8a6e1cbf38cea44be1467053b0aea5bbfcb4b13c4b88c4", @@ -690,6 +544,22 @@ ], "version": "==2.20" }, + "pyfakefs": { + "hashes": [ + "sha256:226e8f141e40c19d10693f8df4968407eb47100705796640ee0dc5990bd6f109", + "sha256:8312f143b37b014cdd4641ac1f4eac0c47bb881774a3d0be125997133e609a6b" + ], + "index": "pypi", + "version": "==3.4.3" + }, + "pylint": { + "hashes": [ + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + ], + "index": "pypi", + "version": "==2.4.4" + }, "pynacl": { "hashes": [ "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", @@ -716,6 +586,30 @@ ], "version": "==1.3.0" }, + "python-mimeparse": { + "hashes": [ + "sha256:bef134a59598cc6aa598f84553162aa7a0c01f3f431588225bb9a208964b1827" + ], + "index": "pypi", + "version": "==1.5.2" + }, + "pyyaml": { + "hashes": [ + "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", + "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", + "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", + "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", + "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", + "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", + "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", + "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", + "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", + "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", + "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" + ], + "index": "pypi", + "version": "==5.1" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -749,6 +643,80 @@ "index": "pypi", "version": "==1.1.0" }, + "testtools": { + "hashes": [ + "sha256:80f606607a6e4ce4d0e24e5b786562aa42c581906f3c070607a4265f3da65810", + "sha256:9b21a293cd33853956b1d3834c294d77a6ad0ab0eb1c077f858be433f0f225bb" + ], + "index": "pypi", + "version": "==2.2.0" + }, + "traceback2": { + "hashes": [ + "sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030", + "sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23" + ], + "index": "pypi", + "version": "==1.4.0" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", + "version": "==1.4.1" + }, + "unittest2": { + "hashes": [ + "sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8", + "sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "waitress": { + "hashes": [ + "sha256:045b3efc3d97c93362173ab1dfc159b52cfa22b46c3334ffc805dbdbf0e4309e", + "sha256:77ff3f3226931a1d7d8624c5371de07c8e90c7e5d80c5cc660d72659aaf23f38" + ], + "index": "pypi", + "version": "==1.4.3" + }, + "webob": { + "hashes": [ + "sha256:a3c89a8e9ba0aeb17382836cdb73c516d0ecf6630ec40ec28288f3ed459ce87b", + "sha256:aa3a917ed752ba3e0b242234b2a373f9c4e2a75d35291dcbe977649bd21fd108" + ], + "version": "==1.8.6" + }, + "webtest": { + "hashes": [ + "sha256:0831371d415fac39177d1fab4e180510713401a71edd9181373efe123e85cb3e", + "sha256:879da5c2310b75c74dffa23a45a7a74d09c2a0fcc7e7a128b4f92a4f64722e97" + ], + "index": "pypi", + "version": "==2.0.23" + }, "werkzeug": { "hashes": [ "sha256:169ba8a33788476292d04186ab33b01d6add475033dfc07215e6d219cc077096", @@ -764,6 +732,21 @@ ], "markers": "python_version >= '3'", "version": "==0.34.2" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + }, + "yapf": { + "hashes": [ + "sha256:6567745f0b6656f9c33a73c56a393071c699e6284a70d793798ab6e3769d25ec", + "sha256:a98a6eacca64d2b920558f4a2f78150db9474de821227e60deaa29f186121c63" + ], + "index": "pypi", + "version": "==0.22.0" } - } + }, + "develop": {} } diff --git a/cloudbuild3.yaml b/cloudbuild3.yaml new file mode 100644 index 0000000000..1c2fb5fa4c --- /dev/null +++ b/cloudbuild3.yaml @@ -0,0 +1,54 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- name: gcr.io/cloud-builders/git + entrypoint: /bin/bash + args: + - -exc + - | + # Cloud Build x GitHub integration uses source archives to fetch + # the source, rather than Git source fetching, and as a consequence + # does not include the .git/ directory. As a workaround, we clone + # the repository to grab the .git directory. + git clone 'https://github.com/google/clusterfuzz' tmp + git -C tmp fetch origin "$COMMIT_SHA" + git -C tmp checkout -qf FETCH_HEAD + mv tmp/.git .git + rm -rf tmp +- name: 'gcr.io/clusterfuzz-images/ci' + args: ['pipenv', 'sync'] + env: + - PIPENV_VENV_IN_PROJECT=1 + +- name: 'gcr.io/clusterfuzz-images/ci' + args: ['pipenv', 'run', 'python', 'butler.py', 'lint'] + env: + - GOOGLE_CLOUDBUILD=1 + - PIPENV_VENV_IN_PROJECT=1 +- name: 'gcr.io/clusterfuzz-images/ci' + args: ['pipenv', 'run', 'setup'] +- name: 'gcr.io/clusterfuzz-images/ci' + args: ['pipenv', 'run', 'local/tests/run_all_tests'] + env: + - TEST_BLOBS_BUCKET=clusterfuzz-ci-blobs + - TEST_BUCKET=clusterfuzz-ci-test + - TEST_CORPUS_BUCKET=clusterfuzz-ci-corpus + - TEST_QUARANTINE_BUCKET=clusterfuzz-ci-quarantine + - TEST_BACKUP_BUCKET=clusterfuzz-ci-backup + - TEST_COVERAGE_BUCKET=clusterfuzz-ci-coverage + - PIPENV_VENV_IN_PROJECT=1 +timeout: 5400s +options: + machineType: N1_HIGHCPU_32 diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index 5d7ecd9816..a9a91d3228 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -30,6 +30,14 @@ RUN apt-get update && \ nodejs \ openjdk-8-jdk +# Install Python 3. +# TODO(ochang): Move this to base image. +RUN curl -sS https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar -C /tmp -xzv && \ + cd /tmp/Python-3.7.7 && \ + ./configure --enable-optimizations && make altinstall && \ + rm -rf /tmp/Python-3.7.7 +RUN pip3.7 install pipenv + # Install Bazel as per https://docs.bazel.build/versions/master/install-ubuntu.html#using-bazel-custom-apt-repository. RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ diff --git a/src/local/butler/integration_tests.py b/src/local/butler/integration_tests.py index f06f5eeb51..fc3c5c9db8 100644 --- a/src/local/butler/integration_tests.py +++ b/src/local/butler/integration_tests.py @@ -15,6 +15,7 @@ from __future__ import print_function from future import standard_library standard_library.install_aliases() +import sys import time import urllib.error import urllib.request @@ -28,14 +29,21 @@ def execute(_): """Run integration tests.""" + if sys.version_info.major == 3: + command = 'run_server3' + indicator = b'Booting worker' + else: + command = 'run_server' + indicator = b'Starting admin server' + try: lines = [] server = common.execute_async( - 'python -u butler.py run_server --skip-install-deps') + 'python -u butler.py {} --skip-install-deps'.format(command)) test_utils.wait_for_emulator_ready( server, - 'run_server', - 'Starting admin server', + command, + indicator, timeout=RUN_SERVER_TIMEOUT, output_lines=lines) @@ -55,7 +63,7 @@ def execute(_): request.read() # Raises exception on error except Exception: print('Error occurred:') - print(''.join(lines)) + print(b''.join(lines)) raise finally: server.terminate()