feat(Response): add Response.fromCache / Response.fromServiceWorker (p…

…uppeteer#1971)

This patch:
- introduces `test/assets/cached` folder and teaches server to cache
  all the assets from the folder
- introduces `test/assets/serviceworkers` folder that stores all the
  service workers and makes them register with unique URL prefix
- introduces `Response.fromCache()` and `Response.fromServiceWorker()`
  methods

Fixes puppeteer#1551.

docs/api.md

@@ -200,6 +200,8 @@
   * [request.url()](#requesturl)
 - [class: Response](#class-response)
   * [response.buffer()](#responsebuffer)
+  * [response.fromCache()](#responsefromcache)
+  * [response.fromServiceWorker()](#responsefromserviceworker)
   * [response.headers()](#responseheaders)
   * [response.json()](#responsejson)
   * [response.ok()](#responseok)
@@ -2319,6 +2321,16 @@ page.on('request', request => {
 #### response.buffer()
 - returns: <Promise<[Buffer]>> Promise which resolves to a buffer with response body.
 
+#### response.fromCache()
+- returns: <[boolean]>
+
+True if the response was served from either the browser's disk cache or memory cache.
+
+#### response.fromServiceWorker()
+- returns: <[boolean]>
+
+True if the response was served by a service worker.
+
 #### response.headers()
 - returns: <[Object]> An object with HTTP headers associated with the response. All header names are lower-case.
 

lib/NetworkManager.js

@@ -48,6 +48,7 @@ class NetworkManager extends EventEmitter {
 
     this._client.on('Network.requestWillBeSent', this._onRequestWillBeSent.bind(this));
     this._client.on('Network.requestIntercepted', this._onRequestIntercepted.bind(this));
+    this._client.on('Network.requestServedFromCache', this._onRequestServedFromCache.bind(this));
     this._client.on('Network.responseReceived', this._onResponseReceived.bind(this));
     this._client.on('Network.loadingFinished', this._onLoadingFinished.bind(this));
     this._client.on('Network.loadingFailed', this._onLoadingFailed.bind(this));
@@ -152,7 +153,7 @@ class NetworkManager extends EventEmitter {
     if (event.redirectUrl) {
       const request = this._interceptionIdToRequest.get(event.interceptionId);
       if (request) {
-        this._handleRequestRedirect(request, event.responseStatusCode, event.responseHeaders);
+        this._handleRequestRedirect(request, event.responseStatusCode, event.responseHeaders, false /* fromDiskCache */, false /* fromServiceWorker */);
         this._handleRequestStart(request._requestId, event.interceptionId, event.redirectUrl, event.resourceType, event.request, event.frameId);
       }
       return;
@@ -168,13 +169,24 @@ class NetworkManager extends EventEmitter {
     }
   }
 
+  /**
+   * @param {!Object} event
+   */
+  _onRequestServedFromCache(event) {
+    const request = this._requestIdToRequest.get(event.requestId);
+    if (request)
+      request._fromMemoryCache = true;
+  }
+
   /**
    * @param {!Request} request
    * @param {number} redirectStatus
    * @param {!Object} redirectHeaders
+   * @param {boolean} fromDiskCache
+   * @param {boolean} fromServiceWorker
    */
-  _handleRequestRedirect(request, redirectStatus, redirectHeaders) {
-    const response = new Response(this._client, request, redirectStatus, redirectHeaders);
+  _handleRequestRedirect(request, redirectStatus, redirectHeaders, fromDiskCache, fromServiceWorker) {
+    const response = new Response(this._client, request, redirectStatus, redirectHeaders, fromDiskCache, fromServiceWorker);
     request._response = response;
     this._requestIdToRequest.delete(request._requestId);
     this._interceptionIdToRequest.delete(request._interceptionId);
@@ -227,7 +239,7 @@ class NetworkManager extends EventEmitter {
       const request = this._requestIdToRequest.get(event.requestId);
       // If we connect late to the target, we could have missed the requestWillBeSent event.
       if (request)
-        this._handleRequestRedirect(request, event.redirectResponse.status, event.redirectResponse.headers);
+        this._handleRequestRedirect(request, event.redirectResponse.status, event.redirectResponse.headers, event.redirectResponse.fromDiskCache, event.redirectResponse.fromServiceWorker);
     }
     this._handleRequestStart(event.requestId, null, event.request.url, event.type, event.request, event.frameId);
   }
@@ -240,7 +252,8 @@ class NetworkManager extends EventEmitter {
     // FileUpload sends a response without a matching request.
     if (!request)
       return;
-    const response = new Response(this._client, request, event.response.status, event.response.headers);
+    const response = new Response(this._client, request, event.response.status, event.response.headers,
+        event.response.fromDiskCache, event.response.fromServiceWorker);
     request._response = response;
     this.emit(NetworkManager.Events.Response, response);
   }
@@ -310,6 +323,8 @@ class Request {
     this._frame = frame;
     for (const key of Object.keys(payload.headers))
       this._headers[key.toLowerCase()] = payload.headers[key];
+
+    this._fromMemoryCache = false;
   }
 
   /**
@@ -483,14 +498,18 @@ class Response {
    * @param {!Request} request
    * @param {number} status
    * @param {!Object} headers
+   * @param {boolean} fromDiskCache
+   * @param {boolean} fromServiceWorker
    */
-  constructor(client, request, status, headers) {
+  constructor(client, request, status, headers, fromDiskCache, fromServiceWorker) {
     this._client = client;
     this._request = request;
     this._contentPromise = null;
 
     this._status = status;
     this._url = request.url();
+    this._fromDiskCache = fromDiskCache;
+    this._fromServiceWorker = fromServiceWorker;
     this._headers = {};
     for (const key of Object.keys(headers))
       this._headers[key.toLowerCase()] = headers[key];
@@ -561,6 +580,20 @@ class Response {
   request() {
     return this._request;
   }
+
+  /**
+   * @return {boolean}
+   */
+  fromCache() {
+    return this._fromDiskCache || this._request._fromMemoryCache;
+  }
+
+  /**
+   * @return {boolean}
+   */
+  fromServiceWorker() {
+    return this._fromServiceWorker;
+  }
 }
 helper.tracePublicAPI(Response);
 

test/assets/cached/one-style.css

@@ -0,0 +1,3 @@
+body {
+  background-color: pink;
+}

test/assets/cached/one-style.html

@@ -0,0 +1,2 @@
+<link rel='stylesheet' href='./one-style.css'>
+<div>hello, world!</div>

test/assets/serviceworkers/empty/sw.html

@@ -0,0 +1,3 @@
+<script>
+  window.registrationPromise = navigator.serviceWorker.register('sw.js');
+</script>

test/assets/serviceworkers/fetch/style.css

@@ -0,0 +1,3 @@
+body {
+    background-color: pink;
+}

test/assets/serviceworkers/fetch/sw.html

@@ -0,0 +1,4 @@
+<link rel="stylesheet" href="./style.css">
+<script>
+  navigator.serviceWorker.register('sw.js');
+</script>

test/assets/serviceworkers/fetch/sw.js

@@ -0,0 +1,3 @@
+self.addEventListener('fetch', event => {
+  event.respondWith(fetch(event.request));
+});

test/server/SimpleServer.js

@@ -68,6 +68,9 @@ class SimpleServer {
     this._server.listen(port);
     this._dirPath = dirPath;
 
+    this._startTime = new Date();
+    this._cachedPathPrefix = null;
+
     /** @type {!Set<!net.Socket>} */
     this._sockets = new Set();
 
@@ -90,6 +93,13 @@ class SimpleServer {
     socket.once('close', () => this._sockets.delete(socket));
   }
 
+  /**
+   * @param {string} pathPrefix
+   */
+  enableHTTPCache(pathPrefix) {
+    this._cachedPathPrefix = pathPrefix;
+  }
+
   /**
    * @param {string} path
    * @param {string} username
@@ -189,15 +199,27 @@ class SimpleServer {
     let pathName = url.parse(request.url).path;
     if (pathName === '/')
       pathName = '/index.html';
-    pathName = path.join(this._dirPath, pathName.substring(1));
+    const filePath = path.join(this._dirPath, pathName.substring(1));
+
+    if (this._cachedPathPrefix !== null && filePath.startsWith(this._cachedPathPrefix)) {
+      if (request.headers['if-modified-since']) {
+        response.statusCode = 304; // not modified
+        response.end();
+        return;
+      }
+      response.setHeader('Cache-Control', 'public, max-age=31536000');
+      response.setHeader('Last-Modified', this._startTime.toString());
+    } else {
+      response.setHeader('Cache-Control', 'no-cache, no-store');
+    }
 
-    fs.readFile(pathName, function(err, data) {
+    fs.readFile(filePath, function(err, data) {
       if (err) {
         response.statusCode = 404;
-        response.end(`File not found: ${pathName}`);
+        response.end(`File not found: ${filePath}`);
         return;
       }
-      response.setHeader('Content-Type', mime.lookup(pathName));
+      response.setHeader('Content-Type', mime.lookup(filePath));
       response.end(data);
     });
   }

test/server/run.js

@@ -19,10 +19,14 @@ const SimpleServer = require('./SimpleServer');
 const port = 8907;
 const httpsPort = 8908;
 const assetsPath = path.join(__dirname, '..', 'assets');
+const cachedPath = path.join(__dirname, '..', 'assets', 'cached');
+
 Promise.all([
   SimpleServer.create(assetsPath, port),
   SimpleServer.createHTTPS(assetsPath, httpsPort)
 ]).then(([server, httpsServer]) => {
+  server.enableHTTPCache(cachedPath);
+  httpsServer.enableHTTPCache(cachedPath);
   console.log(`HTTP: server is running on http://localhost:${port}`);
   console.log(`HTTPS: server is running on https://localhost:${httpsPort}`);
 });

test/test.js

@@ -80,14 +80,18 @@ if (fs.existsSync(OUTPUT_DIR))
 
 beforeAll(async state  => {
   const assetsPath = path.join(__dirname, 'assets');
+  const cachedPath = path.join(__dirname, 'assets', 'cached');
+
   const port = 8907 + state.parallelIndex * 2;
   state.server = await SimpleServer.create(assetsPath, port);
+  state.server.enableHTTPCache(cachedPath);
   state.server.PREFIX = `http://localhost:${port}`;
   state.server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`;
   state.server.EMPTY_PAGE = `http://localhost:${port}/empty.html`;
 
   const httpsPort = port + 1;
   state.httpsServer = await SimpleServer.createHTTPS(assetsPath, httpsPort);
+  state.httpsServer.enableHTTPCache(cachedPath);
   state.httpsServer.PREFIX = `https://localhost:${httpsPort}`;
   state.httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`;
   state.httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`;
@@ -2663,8 +2667,40 @@ describe('Page', function() {
       expect(responses[0].url()).toBe(server.EMPTY_PAGE);
       expect(responses[0].status()).toBe(200);
       expect(responses[0].ok()).toBe(true);
+      expect(responses[0].fromCache()).toBe(false);
+      expect(responses[0].fromServiceWorker()).toBe(false);
       expect(responses[0].request()).toBeTruthy();
     });
+
+    it('Response.fromCache()', async({page, server}) => {
+      const responses = new Map();
+      page.on('response', r => responses.set(r.url().split('/').pop(), r));
+
+      // Load and re-load to make sure it's cached.
+      await page.goto(server.PREFIX + '/cached/one-style.html');
+      await page.reload();
+
+      expect(responses.size).toBe(2);
+      expect(responses.get('one-style.html').status()).toBe(304);
+      expect(responses.get('one-style.html').fromCache()).toBe(false);
+      expect(responses.get('one-style.css').status()).toBe(200);
+      expect(responses.get('one-style.css').fromCache()).toBe(true);
+    });
+    it('Response.fromServiceWorker', async({page, server}) => {
+      const responses = new Map();
+      page.on('response', r => responses.set(r.url().split('/').pop(), r));
+
+      // Load and re-load to make sure serviceworker is installed and running.
+      await page.goto(server.PREFIX + '/serviceworkers/fetch/sw.html', {waitUntil: 'networkidle2'});
+      await page.reload();
+
+      expect(responses.size).toBe(2);
+      expect(responses.get('sw.html').status()).toBe(200);
+      expect(responses.get('sw.html').fromServiceWorker()).toBe(true);
+      expect(responses.get('style.css').status()).toBe(200);
+      expect(responses.get('style.css').fromServiceWorker()).toBe(true);
+    });
+
     it('Page.Events.Response should provide body', async({page, server}) => {
       let response = null;
       page.on('response', r => response = r);
@@ -3527,13 +3563,13 @@ describe('Page', function() {
     it('should report when a service worker is created and destroyed', async({page, server, browser}) => {
       await page.goto(server.EMPTY_PAGE);
       const createdTarget = new Promise(fulfill => browser.once('targetcreated', target => fulfill(target)));
-      const registration = await page.evaluateHandle(() => navigator.serviceWorker.register('sw.js'));
+      await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html');
 
       expect((await createdTarget).type()).toBe('service_worker');
-      expect((await createdTarget).url()).toBe(server.PREFIX + '/sw.js');
+      expect((await createdTarget).url()).toBe(server.PREFIX + '/serviceworkers/empty/sw.js');
 
       const destroyedTarget = new Promise(fulfill => browser.once('targetdestroyed', target => fulfill(target)));
-      await page.evaluate(registration => registration.unregister(), registration);
+      await page.evaluate(() => window.registrationPromise.then(registration => registration.unregister()));
       expect(await destroyedTarget).toBe(await createdTarget);
     });
     it('should report when a target url changes', async({page, server, browser}) => {