tls-offload to Fly.io edge for DoH and DoT

README.md

@@ -128,14 +128,16 @@ The entrypoint for Node and Deno are [`src/server-node.js`](src/server-node.js),
 and both listen for TCP-over-TLS, HTTP/S connections; whereas, the entrypoint for Cloudflare Workers, which only listens over HTTP (cli) or
 over HTTP/S (prod), is [`src/server-workers.js`](src/server-workers.js).
 
-For prod setups on Deno and local (non-prod) setups on Node, the key (private) and cert (public chain)
+For prod setups on Deno and local (non-prod) setups on Node, the `key` (private) and `cert` (public chain)
 files, by default, are read from paths defined in env vars, `TLS_KEY_PATH` and `TLS_CRT_PATH`.
 
-Whilst for prod setup on Node, the key and cert _must_ be
-_base64_ encoded in env var `TLS_CERTKEY` ([ref](https://github.com/serverless-dns/serverless-dns/blob/15f62846/src/core/node/config.js#L61-L82)), like so:
+Whilst for prod setup on Node (on Fly.io), either `TLS_OFFLOAD` must be set to `true` or `key` and `cert` _must_ be
+_base64_ encoded in env var `TLS_CERTKEY` ([ref](https://github.com/serverless-dns/serverless-dns/blob/f57c579/src/core/node/config.js#L61-L92)), like so:
 
 ```bash
-# base64 representation of both key (private) and cert (public chain)
+# EITHER: offload tls to fly.io and set tls_offload to true
+TLS_OFFLOAD="true"
+# OR: base64 representation of both key (private) and cert (public chain)
 TLS_CERTKEY="KEY=b64_key_content\nCRT=b64_cert_content"
 ```
 
@@ -159,9 +161,12 @@ Cloudflare Workers build-time and runtime configurations are defined in [`wrangl
 For Deno Deploy, the code-base is bundled up in a single javascript file with `deno bundle` and then handed off
 to Deno.com.
 
-For Fly.io, which runs Node, the runtime directives are defined in [`fly.toml`](fly.toml), while deploy directives
-are in [`node.Dockerfile`](node.Dockerfile). [`flyctl`](https://fly.io/docs/flyctl) accordingly sets up `serverless-dns`
-on Fly.io's infrastructure.
+For Fly.io, which runs Node, the runtime directives are defined in [`fly.toml`](fly.toml) (used by `dev` and `live` deployment-types),
+while deploy directives are in [`node.Dockerfile`](node.Dockerfile). [`flyctl`](https://fly.io/docs/flyctl) accordingly sets
+up `serverless-dns` on Fly.io's infrastructure.
+
+For deploys offloading TLS termination to Fly.io (`B1` deployment-type), the runtime directives are instead defined in
+[`fly.tls.toml`](fly.tls.toml), which sets up HTTP2 Cleartext and HTTP/1.1 on port 443, and DNS over TCP on port 853.
 
 Ref: _[github/workflows](.github/workflows)_.
 

fly.tls.toml

@@ -0,0 +1,72 @@
+app = ""
+
+kill_signal = "SIGINT"
+kill_timeout = 5
+
+[env]
+# offload TLS to Fly.io's load balancers
+TLS_OFFLOAD = "true"
+DENO_ENV = "production"
+NODE_ENV = "production"
+CLOUD_PLATFORM = "fly"
+LOG_LEVEL = "info"
+CF_BLOCKLIST_URL = "https://dist.rethinkdns.com/blocklists/"
+CF_LATEST_BLOCKLIST_TIMESTAMP = "1649941811557"
+TD_NODE_COUNT = "38362827"
+TD_PARTS = "2"
+CF_DNS_RESOLVER_URL = "https://cloudflare-dns.com/dns-query"
+CF_BLOCKLIST_DOWNLOAD_TIMEOUT = "10000"
+
+[experimental]
+allowed_public_ports = []
+auto_rollback = true
+
+# DNS over HTTP[S]
+[[services]]
+http_checks = []
+internal_port = 8555
+protocol = "tcp"
+script_checks = []
+
+  [services.concurrency]
+  hard_limit = 75
+  soft_limit = 60
+  type = "connections"
+
+  [[services.ports]]
+  handlers = ["tls"]
+  tls_options = { alpn = ["h2", "http/1.1"] }
+  port = 443
+
+  [[services.tcp_checks]]
+  # account for delay due to blocklists download that
+  # happens on process startup: plugin.js:systemReady
+  grace_period = "15s"
+  interval = "30s"
+  restart_limit = 6
+  timeout = "2s"
+
+# DNS over TCP/TLS
+[[services]]
+http_checks = []
+internal_port = 10555
+protocol = "tcp"
+script_checks = []
+
+  [services.concurrency]
+  hard_limit = 75
+  soft_limit = 60
+  type = "connections"
+
+  [[services.ports]]
+  # TODO: ProxyProto v2
+  handlers = ["tls"]
+  port = 853
+
+  [[services.tcp_checks]]
+  # account for delay due to blocklists download that
+  # happens on process startup: plugin.js:systemReady
+  grace_period = "15s"
+  interval = "30s"
+  restart_limit = 6
+  timeout = "2s"

src/commons/envutil.js

@@ -151,17 +151,33 @@ export function isDotOverProxyProto() {
   return envManager.get("DOT_HAS_PROXY_PROTO") || false;
 }
 
+export function isCleartext() {
+  if (!envManager) return false;
+
+  // when connecting to <appname>.fly.dev domains, fly.io edge handles tls;
+  // and so, conns from fly.io edge to app is in cleartext
+  return envManager.get("TLS_OFFLOAD") || false;
+}
+
 // Ports which the services are exposed on. Corresponds to fly.toml ports.
 export function dohBackendPort() {
   return 8080;
 }
 
+export function dohCleartextBackendPort() {
+  return isCleartext() ? 8055 : /* random*/ 0;
+}
+
 export function dotBackendPort() {
   return isDotOverProxyProto() ? 10001 : 10000;
 }
 
 export function dotProxyProtoBackendPort() {
-  return isDotOverProxyProto() ? 10000 : 0;
+  return isDotOverProxyProto() ? 10000 : /* random*/ 0;
+}
+
+export function dotCleartextBackendPort() {
+  return isCleartext() ? 10555 : /* random*/ 0;
 }
 
 export function profileDnsResolves() {

src/commons/util.js

@@ -261,6 +261,10 @@ export function safeBox(fns, arg) {
   return r;
 }
 
+export function isDohGetRequest(queryString) {
+  return queryString && queryString.has("dns");
+}
+
 /**
  * @param {Request} req - Request
  * @return {Boolean}

src/core/env.js

@@ -54,6 +54,12 @@ const defaults = {
     type: "string",
     default: "test/data/tls/dns.rethinkdns.localhost.crt",
   },
+  // indicate if tls termination is offload to an external process; for example
+  // <appname>.fly.dev as primary access-point w fly.io edge terminating tls.
+  TLS_OFFLOAD: {
+    type: "boolean",
+    default: false,
+  },
   // global log level (debug, info, warn, error)
   LOG_LEVEL: {
     type: "string",

src/core/plugin.js

@@ -233,11 +233,11 @@ export default class RethinkPlugin {
     const rxid = this.parameter.get("rxid");
     const isDnsMsg = util.isDnsMsg(request);
     const isGwReq = util.isGatewayRequest(request);
+    let question = null;
 
     io.id(rxid);
 
     this.registerParameter("isDnsMsg", isDnsMsg);
-
     // nothing to do if the current request isn't a dns question
     if (!isDnsMsg) {
       // throw away any request that is not a dns-msg since cc.js
@@ -246,18 +246,21 @@ export default class RethinkPlugin {
       if (!util.isGetRequest(request)) {
         this.log.i(rxid, "not a dns-msg, not a GET req either", request);
         io.hResponse(util.respond405());
+        return;
       }
-      return;
     }
 
+    // else: treat doh as if it was a dns-msg iff "dns" query-string is set
+    question = await extractDnsQuestion(request);
+    if (question == null) return;
+    this.registerParameter("isDnsMsg", true);
+
     if (isGwReq) io.gatewayAnswersOnly(envutil.gwip4(), envutil.gwip6());
 
-    const question = await extractDnsQuestion(request);
     const questionPacket = dnsutil.decode(question);
-
     this.log.d(rxid, "cur-ques", JSON.stringify(questionPacket.questions));
-
     io.decodedDnsPacket = questionPacket;
+
     this.registerParameter("requestDecodedDnsPacket", questionPacket);
     this.registerParameter("requestBodyBuffer", question);
   }

src/plugins/command-control/cc.js

@@ -41,10 +41,6 @@ export class CommandControl {
     return s === "configure" || s === "config";
   }
 
-  isDohGetRequest(queryString) {
-    return queryString && queryString.has("dns");
-  }
-
   userFlag(url, isDnsCmd = false) {
     const emptyFlag = "";
     const p = url.pathname.split("/"); // ex: max.rethinkdns.com/cmd/XYZ
@@ -69,18 +65,16 @@ export class CommandControl {
     return d.length > 1 ? d[0] : emptyFlag;
   }
 
-  async commandOperation(rxid, url, isDnsMsg) {
+  async commandOperation(rxid, url, isDnsCmd) {
     let response = util.emptyResponse();
 
     try {
       const reqUrl = new URL(url);
       const queryString = reqUrl.searchParams;
       const pathSplit = reqUrl.pathname.split("/");
 
-      // FIXME: isDohGetRequest is redundant, simply trust isDnsMsg as-is
-      const isDnsCmd = isDnsMsg || this.isDohGetRequest(queryString);
-
       if (isDnsCmd) {
+        this.log.d(rxid, "cc no-op: dns-msg not cc-msg");
         response.data.stopProcessing = false;
         return response;
       } else {

src/server-node.js

@@ -38,29 +38,53 @@ let log = null;
 })();
 
 function systemUp() {
-  const tlsOpts = {
-    key: envutil.tlsKey(),
-    cert: envutil.tlsCrt(),
-  };
-
   log = util.logger("NodeJs");
   if (!log) throw new Error("logger unavailable on system up");
 
-  const dot1 = tls
-    .createServer(tlsOpts, serveTLS)
-    .listen(envutil.dotBackendPort(), () => up("DoT", dot1.address()));
+  const tlsoffload = envutil.isCleartext() || true;
+
+  if (tlsoffload) {
+    // fly.io terminated tls?
+    const portdoh = envutil.dohCleartextBackendPort();
+    const portdot = envutil.dotCleartextBackendPort();
 
-  const dot2 =
-    envutil.isDotOverProxyProto() &&
-    net
-      .createServer(serveDoTProxyProto)
-      .listen(envutil.dotProxyProtoBackendPort(), () =>
-        up("DoT ProxyProto", dot2.address())
-      );
+    // TODO: ProxyProtoV2 with TLS ClientHello (unsupported by Fly.io, rn)
+    // DNS over TLS Cleartext
+    const dotct = net
+      .createServer(serveTCP)
+      .listen(portdot, () => up("DoT Cleartext", dotct.address()));
 
-  const doh = http2
-    .createSecureServer({ ...tlsOpts, allowHTTP1: true }, serveHTTPS)
-    .listen(envutil.dohBackendPort(), () => up("DoH", doh.address()));
+    // DNS over HTTPS Cleartext
+    const dohct = http2
+      .createServer({ allowHTTP1: true }, serveHTTPS)
+      .listen(portdoh, () => up("DoH Cleartext", dohct.address()));
+  } else {
+    // terminate tls ourselves
+    const tlsOpts = {
+      key: envutil.tlsKey(),
+      cert: envutil.tlsCrt(),
+    };
+    const portdot1 = envutil.dotBackendPort();
+    const portdot2 = envutil.dotProxyProtoBackendPort();
+    const portdoh = envutil.dohBackendPort();
+
+    // DNS over TLS
+    const dot1 = tls
+      .createServer(tlsOpts, serveTLS)
+      .listen(portdot1, () => up("DoT", dot1.address()));
+
+    // DNS over TLS w ProxyProto
+    const dot2 =
+      envutil.isDotOverProxyProto() &&
+      net
+        .createServer(serveDoTProxyProto)
+        .listen(portdot2, () => up("DoT ProxyProto", dot2.address()));
+
+    // DNS over HTTPS
+    const doh = http2
+      .createSecureServer({ ...tlsOpts, allowHTTP1: true }, serveHTTPS)
+      .listen(portdoh, () => up("DoH", doh.address()));
+  }
 
   function up(server, addr) {
     log.i(server, `listening on: [${addr.address}]:${addr.port}`);
@@ -272,6 +296,30 @@ function serveTLS(socket) {
   });
 }
 
+/**
+ * Services a DNS over TCP connection
+ * @param {Socket} socket
+ */
+function serveTCP(socket) {
+  // TODO: TLS ClientHello is sent in proxy-proto v2, but fly.io
+  // doesn't yet support v2, but only v1. ClientHello would contain
+  // the SNI which we could then use here.
+  const [flag, host] = ["", "ignored.example.com"];
+  const sb = makeScratchBuffer();
+
+  log.d("----> DoT Cleartext request", host, flag);
+  socket.on("data", (data) => {
+    handleTCPData(socket, data, sb, host, flag);
+  });
+  socket.on("end", () => {
+    socket.end();
+  });
+  socket.on("error", (e) => {
+    log.w("TCP socket error, closing connection");
+    close(socket);
+  });
+}
+
 /**
  * Handle DNS over TCP/TLS data stream.
  * @param {TLSSocket} socket