Bug 1693993 - [marionette] Move Marionette server code to /remote/mar…

…ionette. r=remote-protocol-reviewers,marionette-reviewers,jdescottes,webdriver-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D106311

.eslintignore

@@ -155,6 +155,7 @@ python/
 # These are (mainly) imported code that we don't want to lint to make imports easier.
 remote/cdp/Protocol.jsm
 remote/cdp/test/browser/chrome-remote-interface.js
+remote/marionette/atom.js
 
 # services/ exclusions
 
@@ -170,7 +171,6 @@ servo/
 
 # Test files that we don't want to lint (preprocessed, minified etc)
 testing/condprofile/condprof/tests/profile
-testing/marionette/atom.js
 testing/mozbase/mozprofile/tests/files/prefs_with_comments.js
 testing/talos/talos/scripts/jszip.min.js
 testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js

docs/conf.py

@@ -55,7 +55,7 @@
 js_source_path = [
     "../browser/components/extensions",
     "../browser/components/uitour",
-    "../testing/marionette",
+    "../remote/marionette",
     "../toolkit/components/extensions",
     "../toolkit/components/extensions/parent",
     "../toolkit/components/featuregates",

modules/libpref/init/all.js

@@ -4475,6 +4475,16 @@ pref("services.common.log.logger.tokenserverclient", "Debug");
   // Sets recommended automation preferences when Marionette is started.
   pref("marionette.prefs.recommended", true);
 
+  // Defines the protocols that will be active for the Remote Agent.
+  // 1: WebDriver BiDi
+  // 2: CDP (Chrome DevTools Protocol)
+  // 3: WebDriver BiDi + CDP
+  #if defined(NIGHTLY_BUILD)
+    pref("remote.active-protocols", 3);
+  #else
+    pref("remote.active-protocols", 2);
+  #endif
+
   // Limits remote agent to listen on loopback devices,
   // e.g. 127.0.0.1, localhost, and ::1.
   pref("remote.force-local", true);

remote/components/RemoteAgent.jsm

@@ -23,13 +23,31 @@ XPCOMUtils.defineLazyModuleGetters(this, {
 
 XPCOMUtils.defineLazyGetter(this, "logger", () => Log.get());
 
-const FORCE_LOCAL = "remote.force-local";
+const PREF_ACTIVE_PROTOCOLS = ["remote.active-protocols"];
+const PREF_FORCE_LOCAL = "remote.force-local";
+
+const BIDI_ACTIVE = 0x1;
+const CDP_ACTIVE = 0x2;
 
 const LOOPBACKS = ["localhost", "127.0.0.1", "[::1]"];
 
 class RemoteAgentClass {
   constructor() {
     this.alteredPrefs = new Set();
+
+    const protocols = Services.prefs.getIntPref(PREF_ACTIVE_PROTOCOLS);
+    if (protocols < 1 || protocols > 3) {
+      throw Error(`Invalid remote protocol identifier: ${protocols}`);
+    }
+    this._activeProtocols = protocols;
+  }
+
+  get isBiDiEnabled() {
+    return (this._activeProtocols & BIDI_ACTIVE) == BIDI_ACTIVE;
+  }
+
+  get isCDPEnabled() {
+    return (this._activeProtocols & CDP_ACTIVE) == CDP_ACTIVE;
   }
 
   get listening() {
@@ -61,7 +79,7 @@ class RemoteAgentClass {
     }
 
     let { host, port } = url;
-    if (Preferences.get(FORCE_LOCAL) && !LOOPBACKS.includes(host)) {
+    if (Preferences.get(PREF_FORCE_LOCAL) && !LOOPBACKS.includes(host)) {
       throw Components.Exception(
         "Restricted to loopback devices",
         Cr.NS_ERROR_ILLEGAL_VALUE
@@ -82,33 +100,39 @@ class RemoteAgentClass {
     }
 
     this.server = new HttpServer();
-    this.server.registerPrefixHandler("/json/", new JSONHandler(this));
 
-    this.targetList = new TargetList();
-    this.targetList.on("target-created", (eventName, target) => {
-      this.server.registerPathHandler(target.path, target);
-    });
-    this.targetList.on("target-destroyed", (eventName, target) => {
-      this.server.registerPathHandler(target.path, null);
-    });
+    if (this.isCDPEnabled) {
+      this.server.registerPrefixHandler("/json/", new JSONHandler(this));
+
+      this.targetList = new TargetList();
+      this.targetList.on("target-created", (eventName, target) => {
+        this.server.registerPathHandler(target.path, target);
+      });
+      this.targetList.on("target-destroyed", (eventName, target) => {
+        this.server.registerPathHandler(target.path, null);
+      });
+    }
 
     return this.asyncListen(host, port);
   }
 
   async asyncListen(host, port) {
     try {
-      await this.targetList.watchForTargets();
+      this.server._start(port, host);
 
-      // Immediatly instantiate the main process target in order
-      // to be accessible via HTTP endpoint on startup
-      const mainTarget = this.targetList.getMainProcessTarget();
+      if (this.isCDPEnabled) {
+        await this.targetList.watchForTargets();
 
-      this.server._start(port, host);
-      Services.obs.notifyObservers(
-        null,
-        "remote-listening",
-        `DevTools listening on ${mainTarget.wsDebuggerURL}`
-      );
+        // Immediatly instantiate the main process target in order
+        // to be accessible via HTTP endpoint on startup
+        const mainTarget = this.targetList.getMainProcessTarget();
+
+        Services.obs.notifyObservers(
+          null,
+          "remote-listening",
+          `DevTools listening on ${mainTarget.wsDebuggerURL}`
+        );
+      }
     } catch (e) {
       await this.close();
       logger.error(`Unable to start remote agent: ${e.message}`, e);
@@ -123,10 +147,12 @@ class RemoteAgentClass {
       }
       this.alteredPrefs.clear();
 
-      // destroy targetList before stopping server,
-      // otherwise the HTTP will fail to stop
-      if (this.targetList) {
-        this.targetList.destructor();
+      if (this.isCDPEnabled) {
+        // destroy targetList before stopping server,
+        // otherwise the HTTP will fail to stop
+        if (this.targetList) {
+          this.targetList.destructor();
+        }
       }
 
       if (this.listening) {

remote/components/moz.build

@@ -6,7 +6,21 @@ DIRS += [
     "rust",
 ]
 
+EXTRA_COMPONENTS += [
+    "marionette.js",
+    "marionette.manifest",
+]
+
 XPIDL_MODULE = "remote"
-XPIDL_SOURCES += ["nsIRemoteAgent.idl"]
+
+XPIDL_SOURCES += [
+    "nsIMarionette.idl",
+    "nsIRemoteAgent.idl",
+]
 
 XPCOM_MANIFESTS += ["components.conf"]
+
+with Files("marionette.*"):
+    BUG_COMPONENT = ("Testing", "Marionette")
+with Files("nsIMarionette.idl"):
+    BUG_COMPONENT = ("Testing", "Marionette")

testing/marionette/README → remote/marionette/README

@@ -11,10 +11,10 @@ Marionette [ ˌmarɪəˈnɛt] is
 
 Marionette provides interfaces for interacting with both the internal
 JavaScript runtime and UI elements of Gecko-based browsers, such
-as Firefox and Fennec.  It can control both the chrome- and content
+as Firefox on desktop and mobile.  It can control both the chrome- and content
 documents, giving a high level of control and ability to replicate,
 or emulate, user interaction.
 
 Head on to the Marionette documentation to find out more:
 
-	https://firefox-source-docs.mozilla.org/testing/marionette/marionette/
+	https://firefox-source-docs.mozilla.org/testing/marionette/

testing/marionette/doc/CodeStyle.md → remote/marionette/doc/CodeStyle.md

@@ -14,7 +14,7 @@ For the overall Marionette project, a few rough rules are:
 
   * Code is mutable and not written in stone.  Nothing that
     is checked in is sacred and we encourage change to make
-    testing/marionette a pleasant ecosystem to work in.
+    remote/marionette a pleasant ecosystem to work in.
 
 
 JavaScript
@@ -166,14 +166,14 @@ to make this happen!
 
 The practical details of working on the Marionette code is outlined
 in [CONTRIBUTING.md], but generally you do not have to re-build
-Firefox when changing code.  Any change to testing/marionette/*.js
+Firefox when changing code.  Any change to remote/marionette/*.js
 will be picked up on restarting Firefox.  The only notable exception
-is testing/marionette/components/marionette.js, which does require
+is remote/components/marionette.js, which does require
 a re-build.
 
 [XPCOM]: https://developer.mozilla.org/en-US/docs/Mozilla/Tech/XPCOM
 [strict mode]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Strict_mode
-[our own specialisations]: https://searchfox.org/mozilla-central/source/testing/marionette/.eslintrc.js
+[our own specialisations]: https://searchfox.org/mozilla-central/source/remote/marionette/.eslintrc.js
 [linter]: #linting
 [copying header]: https://www.mozilla.org/en-US/MPL/headers/
 [public domain]: https://creativecommons.org/publicdomain/zero/1.0/
@@ -190,7 +190,7 @@ TODO
 Documentation
 -------------
 
-We keep our documentation in-tree under [testing/marionette/doc]
+We keep our documentation in-tree under [remote/marionette/doc]
 and [testing/geckodriver/doc].  Updates and minor changes to
 documentation should ideally not be scrutinised to the same degree
 as code changes to encourage frequent updates so that the documentation
@@ -207,16 +207,17 @@ These include public functions—or command implementations—on
 the `GeckoDriver` class, as well as all exported symbols from
 other modules.  Documentation for non-exported symbols is not required.
 
-The API documentation can be regenerated to [testing/marionette/doc/api]
+The API documentation can be regenerated to [remote/marionette/doc/internals]
 so:
 
-The API documentation uses [jsdoc] and is generated to <https://firefox-source-docs.mozilla.org/testing/marionette/marionette/internals> on Taskcluster.  You may also build the documentation locally:
+The API documentation uses [jsdoc] and is generated to <https://firefox-source-docs.mozilla.org/testing/marionette/internals> on Taskcluster.  You may also build the documentation locally:
 
 	% ./mach doc
 
 [Mozilla eslint rules]: https://searchfox.org/mozilla-central/source/.eslintrc.js
+[remote/marionette/doc]: https://searchfox.org/mozilla-central/source/remote/marionette/doc
+[remote/marionette/doc/internals]: https://searchfox.org/mozilla-central/source/remote/marionette/doc/internals
 [testing/geckodriver/doc]: https://searchfox.org/mozilla-central/source/testing/geckodriver/doc
-[testing/marionette/doc]: https://searchfox.org/mozilla-central/source/testing/marionette/doc
 [jsdoc]: http://usejsdoc.org/
 
 
@@ -229,13 +230,13 @@ which harmonises the output from [eslint] and [flake8].
 
 To run the linter with a sensible output:
 
-	% ./mach lint -funix testing/marionette
+	% ./mach lint -funix remote/marionette
 
 For certain classes of style violations the eslint linter has
 an automatic mode for fixing and formatting your code.  This is
 particularly useful to keep to whitespace and indentation rules:
 
-	% ./mach eslint --fix testing/marionette
+	% ./mach eslint --fix remote/marionette
 
 The linter is also run as a try job (shorthand `ES`) which means
 any style violations will automatically block a patch from landing

testing/marionette/doc/Contributing.md → remote/marionette/doc/Contributing.md

@@ -56,7 +56,7 @@ code in our [code style document], which we highly recommend you read.
 
 [ask questions]: ./index.html#communication
 [reach out to us]: ./index.html#communication
-[mozilla-central]: https://searchfox.org/mozilla-central/source/testing/marionette/
+[mozilla-central]: https://searchfox.org/mozilla-central/source/remote/marionette/
 [good first bugs]: https://codetribute.mozilla.org/projects/automation
 [code style document]: CodeStyle.html
 

testing/marionette/doc/NewContributors.md → remote/marionette/doc/NewContributors.md

@@ -64,7 +64,8 @@ your patches are laid out in [Patches.md](Patches.html).
 Resources
 ---------
 
-  * Search Mozilla's code repositories with [searchfox].
+  * Search Mozilla's code repository with searchfox to find the [code for
+    Marionette] and the [Marionette client/harness].
 
   * Another [guide for new contributors].  It has not been updated in a long
     time but it's a good general resource if you ever get stuck on something.
@@ -81,7 +82,8 @@ Resources
   * If you'd rather use git instead of hg, see [git workflow for
     Gecko development] and/or [this blog post by :ato].
 
-[searchfox]: https://searchfox.org/mozilla-central/source/testing/marionette/
+[code for Marionette]: https://searchfox.org/mozilla-central/source/remote/marionette/
+[Marionette client/harness]: https://searchfox.org/mozilla-central/source/testing/marionette/
 [guide for new contributors]: https://ateam-bootcamp.readthedocs.org/en/latest/guide/index.html#new-contributor-guide
 [Mercurial for Mozillians]: https://mozilla-version-control-tools.readthedocs.org/en/latest/hgmozilla/index.html
 [guide]: https://gist.github.com/mjzffr/d2adef328a416081f543

testing/marionette/doc/SeleniumAtoms.md → remote/marionette/doc/SeleniumAtoms.md

@@ -17,7 +17,7 @@ To use one of those atoms Javascript modules will have to import
 
 [Selenium atoms]: https://github.com/SeleniumHQ/selenium/tree/master/javascript/webdriver/atoms
 [WebDriver specification]: https://w3c.github.io/webdriver/webdriver-spec.html
-[atom.js]: https://searchfox.org/mozilla-central/source/testing/marionette/atom.js
+[atom.js]: https://searchfox.org/mozilla-central/source/remote/marionette/atom.js
 
 
 Update required Selenium atoms

testing/marionette/doc/Testing.md → remote/marionette/doc/Testing.md

@@ -23,14 +23,14 @@ xpcshell unit tests
 -------------------
 
 Marionette has a set of [xpcshell] unit tests located in
-_testing/marionette/test/unit.  These can be run this way:
+_remote/marionette/test/xpcshell.  These can be run this way:
 
-	% ./mach test testing/marionette/test/unit
+	% ./mach test remote/marionette/test/unit
 
 Because tests are run in parallel and xpcshell itself is quite
 chatty, it can sometimes be useful to run the tests sequentially:
 
-	% ./mach test --sequential testing/marionette/test/unit/test_error.js
+	% ./mach test --sequential remote/marionette/test/xpcshell/test_error.js
 
 These unit tests run as part of the `X` jobs on Treeherder.
 

testing/marionette/jar.mn → remote/marionette/jar.mn

@@ -39,7 +39,7 @@ marionette.jar:
   content/prefs.js (prefs.js)
   content/print.js (print.js)
   content/reftest.js (reftest.js)
-  content/reftest.xhtml (reftest.xhtml)
+  content/reftest.xhtml (chrome/reftest.xhtml)
   content/reftest-content.js (reftest-content.js)
   content/server.js (server.js)
   content/session.js (session.js)

remote/marionette/moz.build

@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+JAR_MANIFESTS += ["jar.mn"]
+
+XPCSHELL_TESTS_MANIFESTS += ["test/xpcshell/xpcshell.ini"]
+
+with Files("**"):
+    BUG_COMPONENT = ("Testing", "Marionette")
+
+SPHINX_TREES["/testing/marionette"] = "doc"
+
+with Files("doc/**"):
+    SCHEDULES.exclusive = ["docs"]

testing/marionette/test/unit/README → remote/marionette/test/xpcshell/README

@@ -1,11 +1,11 @@
 To run the tests in this directory, from the top source directory,
 either invoke the test despatcher in mach:
 
-	% ./mach test testing/marionette/test/unit
+	% ./mach test remote/marionette/test/xpcshell
 
 Or call out the harness specifically:
 
-	% ./mach xpcshell-test testing/marionette/test/unit
+	% ./mach xpcshell-test remote/marionette/test/xpcshell
 
 The latter gives you the --sequential option which can be useful
 when debugging to prevent tests from running in parallel.

remote/moz.build

@@ -5,6 +5,7 @@
 DIRS += [
     "cdp",
     "components",
+    "marionette",
     "shared",
 ]
 

testing/geckodriver/doc/Profiles.md

@@ -75,7 +75,7 @@ Firefox (1), and a set of recommended preferences set on startup (2).
 These can be perused here:
 
   1. [testing/geckodriver/src/prefs.rs](https://searchfox.org/mozilla-central/source/testing/geckodriver/src/prefs.rs)
-  2. [testing/marionette/components/marionette/marionette.js](https://searchfox.org/mozilla-central/source/testing/marionette/components/marionette.js)
+  2. [remote/components/marionette.js](https://searchfox.org/mozilla-central/source/remote/components/marionette.js)
 
 As mentioned, these are _recommended_ preferences, and any user-defined
 preferences in the [user.js file] or as part of the [`prefs` capability]