Converting envoy configs to V2 (envoyproxy#2957)

Signed-off-by: sri kailash <[email protected]>

bazel/repositories.bzl

@@ -189,6 +189,7 @@ def _python_deps():
         build_file = "@envoy//bazel/external:twitter_common_finagle_thrift.BUILD",
     )
 
+
 # Bazel native C++ dependencies. For the depedencies that doesn't provide autoconf/automake builds.
 def _cc_deps():
     _repository_impl("grpc_httpjson_transcoding")

configs/BUILD

@@ -13,8 +13,8 @@ envoy_py_test_binary(
     name = "configgen",
     srcs = ["configgen.py"],
     data = glob([
-        "*.json",
         "*.yaml",
+        "*.json",
     ]),
     external_deps = ["jinja2"],
 )

configs/access_log_format_helper_v2.template.yaml

@@ -0,0 +1,15 @@
+{% macro ingress_sampled_log() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
+{% endmacro %}
+
+{% macro ingress_full() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
+{% endmacro %}
+
+{% macro egress_error_log() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"
+{% endmacro %}
+
+{% macro egress_error_amazon_service() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%RESP(X-AMZN-RequestId)%\"\n"
+{% endmacro %}

configs/configgen.py

@@ -49,11 +49,15 @@
 external_virtual_hosts = [
 {
     'name': 'dynamodb_iad',
-    'address': "tcp://127.0.0.1:9204",
+    'address': "127.0.0.1",
+    'protocol':"TCP",
+    'port_value':"9204",
     'hosts': [
         {
             'name': 'dynamodb_iad', 'domain': '*',
-            'remote_address': 'dynamodb.us-east-1.amazonaws.com:443',
+            'remote_address': 'dynamodb.us-east-1.amazonaws.com',
+            'protocol': 'TCP',
+            'port_value': '443',
             'verify_subject_alt_name': [ 'dynamodb.us-east-1.amazonaws.com' ],
             'ssl': True
         }
@@ -69,12 +73,14 @@
 # as it demonstrates how to setup TCP proxy and the network rate limit filter.
 mongos_servers = {
     'somedb': {
-        'address': "tcp://127.0.0.1:27019",
+        'address': "127.0.0.1",
+        'protocol': "TCP",
+        'port_value': 27019,
         'hosts': [
-            "router1.yourcompany.net:27817",
-            "router2.yourcompany.net:27817",
-            "router3.yourcompany.net:27817",
-            "router4.yourcompany.net:27817",
+            {'port_value' : 27817, 'address':'router1.yourcompany.net' , 'protocol': 'TCP'},
+            {'port_value' : 27817, 'address':'router2.yourcompany.net' , 'protocol': 'TCP'},
+            {'port_value' : 27817, 'address':'router3.yourcompany.net' , 'protocol': 'TCP'},
+            {'port_value' : 27817, 'address':'router4.yourcompany.net' , 'protocol': 'TCP'},
         ],
         'ratelimit': True
     }
@@ -85,20 +91,18 @@ def generate_config(template_path, template, output_file, **context):
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path, followlinks=True),
                              undefined=jinja2.StrictUndefined)
     raw_output = env.get_template(template).render(**context)
-    # Verify valid JSON and then dump it nicely formatted to avoid jinja pain.
-    output = json.loads(raw_output, object_pairs_hook=OrderedDict)
     with open(output_file, 'w') as fh:
-        json.dump(output, fh, indent=2)
+        fh.write(raw_output)
 
 # Generate a demo config for the main front proxy. This sets up both HTTP and HTTPS listeners,
 # as well as a listener for the double proxy to connect to via SSL client authentication.
-generate_config(SCRIPT_DIR, 'envoy_front_proxy.template.json',
-                '{}/envoy_front_proxy.json'.format(OUT_DIR), clusters=front_envoy_clusters)
+generate_config(SCRIPT_DIR, 'envoy_front_proxy_v2.template.yaml',
+                '{}/envoy_front_proxy.v2.yaml'.format(OUT_DIR), clusters=front_envoy_clusters)
 
 # Generate a demo config for the double proxy. This sets up both an HTTP and HTTPS listeners,
 # and backhauls the traffic to the main front proxy.
-generate_config(SCRIPT_DIR, 'envoy_double_proxy.template.json',
-                '{}/envoy_double_proxy.json'.format(OUT_DIR))
+generate_config(SCRIPT_DIR, 'envoy_double_proxy_v2.template.yaml',
+                '{}/envoy_double_proxy.v2.yaml'.format(OUT_DIR))
 
 # Generate a demo config for the service to service (local) proxy. This sets up several different
 # listeners:
@@ -108,11 +112,11 @@ def generate_config(template_path, template, output_file, **context):
 # optional external service ports: built from external_virtual_hosts above. Each external host
 #                                  that Envoy proxies to listens on its own port.
 # optional mongo ports: built from mongos_servers above.
-generate_config(SCRIPT_DIR, 'envoy_service_to_service.template.json',
-                '{}/envoy_service_to_service.json'.format(OUT_DIR),
+generate_config(SCRIPT_DIR, 'envoy_service_to_service_v2.template.yaml',
+                '{}/envoy_service_to_service.yaml'.format(OUT_DIR),
                 internal_virtual_hosts=service_to_service_envoy_clusters,
                 external_virtual_hosts=external_virtual_hosts,
                 mongos_servers=mongos_servers)
 
-for google_ext in ['json', 'yaml', 'v2.yaml']:
+for google_ext in ['v2.yaml']:
   shutil.copy(os.path.join(SCRIPT_DIR, 'google_com_proxy.%s' % google_ext), OUT_DIR)