Add Tool support for both 32 bit (3.2) and 64 bit (3.4) Mongo

These changes introduce dual Mongo support into the Meteor
Tool. 32-bit Mongo (3.2.15) will be used by Meteor when the
Tool is run on a 32-bit OS (32-bit Linux and Windows). 64-bit
Mongo (3.4.9) will be used when the Tool is run on a 64-bit
OS (64-bit Linux, Windows and macOS).

Fixes meteor/meteor-feature-requests#129.

History.md

@@ -6,6 +6,15 @@
   This is a *major* upgrade from the previous version of Node.js used by
   Meteor, 4.8.4.
 
+* Multiple architecture Mongo support has been added to the Meteor Tool.
+  32-bit versions of Mongo have been discontinued as of Mongo 3.4, so to
+  allow people running Meteor on a 64-bit OS to use Mongo 3.4, the Tool has
+  been updated to support both 32 and 64 bit versions of Mongo. Meteor
+  running on a 32-bit OS is limited to Mongo 3.2.15, whereas Meteor running
+  on a 64-bit OS uses Mongo 3.4.9 by default.
+  [FR #129](https://github.com/meteor/meteor-feature-requests/issues/129)
+  [PR TODO]()
+
 * The `npm` npm package has been upgraded to version 5.4.2, a major
   upgrade from 4.6.1, requiring internal updates to dependency management
   logic for Meteor packages that use `Npm.depends`. While these changes

scripts/admin/test-packages-with-mongo-versions.rb

@@ -7,10 +7,10 @@
 require 'tmpdir'
 
 mongo_install_urls = {
-  "3.2.6" => "https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.2.6.tgz",
+  "3.4.9" => "https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.4.9.tgz",
+  "3.2.15" => "https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.2.15.tgz",
   "3.0.5" => "https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.0.5.tgz",
-  "2.6.10" => "http://downloads.mongodb.org/osx/mongodb-osx-x86_64-2.6.10.tgz",
-  "2.4.14" => "http://downloads.mongodb.org/osx/mongodb-osx-x86_64-2.4.14.tgz"
+  "2.6.10" => "http://downloads.mongodb.org/osx/mongodb-osx-x86_64-2.6.10.tgz"
 }
 
 mongo_port = "12345"
@@ -29,7 +29,7 @@
 
 test_env = "TEST_PACKAGES_EXCLUDE=\"less\""
 
-["3.2.6", "3.0.5", "2.6.10", "2.4.14"].each do |mongo_version|
+["3.4.9", "3.2.15", "3.0.5", "2.6.10"].each do |mongo_version|
   puts "Installing and testing with Mongo #{mongo_version}..."
 
   Dir.mktmpdir "mongo_install" do |mongo_install_dir|

scripts/build-dev-bundle-common.sh

@@ -7,6 +7,8 @@ UNAME=$(uname)
 ARCH=$(uname -m)
 MONGO_VERSION=3.2.15
 NODE_VERSION=8.7.0
+MONGO_VERSION_64BIT=3.4.9
+MONGO_VERSION_32BIT=3.2.15
 NPM_VERSION=5.4.2
 
 # If we built Node from source on Jenkins, this is the build number.

scripts/generate-dev-bundle.ps1

@@ -23,7 +23,8 @@ function Get-ShellScriptVariableFromFile {
 $BUNDLE_VERSION = Get-ShellScriptVariableFromFile -Path "${CHECKOUT_DIR}\meteor" -Name 'BUNDLE_VERSION'
 
 # extract the major package versions from the build-dev-bundle-common script.
-$MONGO_VERSION = Get-ShellScriptVariableFromFile -Path $common_script -Name 'MONGO_VERSION'
+$MONGO_VERSION_64BIT = Get-ShellScriptVariableFromFile -Path $common_script -Name 'MONGO_VERSION_64BIT'
+$MONGO_VERSION_32BIT = Get-ShellScriptVariableFromFile -Path $common_script -Name 'MONGO_VERSION_32BIT'
 $NODE_VERSION = Get-ShellScriptVariableFromFile -Path $common_script -Name 'NODE_VERSION'
 $NPM_VERSION = Get-ShellScriptVariableFromFile -Path $common_script -Name 'NPM_VERSION'
 
@@ -165,31 +166,37 @@ cmd /c robocopy "${DIR}\b\p\node_modules" "${DIR}\lib\node_modules" /e /nfl /ndl
 cd "$DIR"
 cmd /c rmdir "${DIR}\b" /s /q
 
-cd "$DIR"
-mkdir "$DIR\mongodb"
-mkdir "$DIR\mongodb\bin"
-
-# download Mongo
-$mongo_name = "mongodb-win32-i386-${MONGO_VERSION}"
-If ($PLATFORM -eq 'windows_x86_64') {
-  # 64-bit would be mongodb-win32-x86_64-2008plus-${MONGO_VERSION}.zip
-  $mongo_name = "mongodb-win32-x86_64-2008plus-${MONGO_VERSION}"
+# Download and install both 32-bit (i386) and 64-bit (x86_64) versions of
+# Mongo. Even though we're currently only building 32-bit Windows Meteor
+# bundles, here we're adding both 32 and 64 bit Mongo versions, to let 64-bit
+# Windows users use the Meteor Tool with >= 3.4.x versions of Mongo (which
+# are 64-bit only).
+$mongo_filenames = @{
+  i386 = "mongodb-win32-i386-${MONGO_VERSION_32BIT}";
+  x86_64 = "mongodb-win32-x86_64-2008plus-${MONGO_VERSION_64BIT}"
 }
-$mongo_link = "https://fastdl.mongodb.org/win32/${mongo_name}.zip"
-$mongo_zip = "$DIR\mongodb\mongo.zip"
+foreach ($arch in $mongo_filenames.Keys) {
+  cd "$DIR"
+  mkdir "$DIR\mongodb\$arch"
+  mkdir "$DIR\mongodb\$arch\bin"
 
-$webclient.DownloadFile($mongo_link, $mongo_zip)
+  $mongo_name = $mongo_filenames.Item($arch);
+  $mongo_link = "https://fastdl.mongodb.org/win32/${mongo_name}.zip"
+  $mongo_zip = "$DIR\mongodb\$arch\mongo.zip"
 
-$zip = $shell.NameSpace($mongo_zip)
-foreach($item in $zip.items()) {
-  $shell.Namespace("$DIR\mongodb").copyhere($item, 0x14) # 0x10 - overwrite, 0x4 - no dialog
-}
+  $webclient.DownloadFile($mongo_link, $mongo_zip)
 
-cp "$DIR\mongodb\$mongo_name\bin\mongod.exe" $DIR\mongodb\bin
-cp "$DIR\mongodb\$mongo_name\bin\mongo.exe" $DIR\mongodb\bin
+  $zip = $shell.NameSpace($mongo_zip)
+  foreach($item in $zip.items()) {
+    $shell.Namespace("$DIR\mongodb\$arch").copyhere($item, 0x14) # 0x10 - overwrite, 0x4 - no dialog
+  }
 
-rm -Recurse -Force $mongo_zip
-rm -Recurse -Force "$DIR\mongodb\$mongo_name"
+  cp "$DIR\mongodb\$arch\$mongo_name\bin\mongod.exe" $DIR\mongodb\$arch\bin
+  cp "$DIR\mongodb\$arch\$mongo_name\bin\mongo.exe" $DIR\mongodb\$arch\bin
+
+  rm -Recurse -Force $mongo_zip
+  rm -Recurse -Force "$DIR\mongodb\$arch\$mongo_name"
+}
 
 cd $DIR
 

scripts/generate-dev-bundle.sh

@@ -44,17 +44,23 @@ downloadOfficialNode() {
 # Try each strategy in the following order:
 extractNodeFromTarGz || downloadNodeFromS3 || downloadOfficialNode
 
-# Download Mongo from mongodb.com
+# Download Mongo from mongodb.com. Will download a 64-bit version of Mongo
+# by default. Will download a 32-bit version of Mongo if using a 32-bit based
+# OS.
+MONGO_VERSION=$MONGO_VERSION_64BIT
+if [ $ARCH = "i686" ]; then
+  MONGO_VERSION=$MONGO_VERSION_32BIT
+fi
 MONGO_NAME="mongodb-${OS}-${ARCH}-${MONGO_VERSION}"
 MONGO_TGZ="${MONGO_NAME}.tgz"
 MONGO_URL="http://fastdl.mongodb.org/${OS}/${MONGO_TGZ}"
 echo "Downloading Mongo from ${MONGO_URL}"
 curl "${MONGO_URL}" | tar zx
 
 # Put Mongo binaries in the right spot (mongodb/bin)
-mkdir -p mongodb/bin
-mv "${MONGO_NAME}/bin/mongod" mongodb/bin
-mv "${MONGO_NAME}/bin/mongo" mongodb/bin
+mkdir -p "mongodb/${ARCH}/bin"
+mv "${MONGO_NAME}/bin/mongod" "mongodb/${ARCH}/bin"
+mv "${MONGO_NAME}/bin/mongo" "mongodb/${ARCH}/bin"
 rm -rf "${MONGO_NAME}"
 
 # export path so we use the downloaded node and npm

tools/packaging/warehouse.js

@@ -42,9 +42,10 @@ var _ = require("underscore");
 var files = require('../fs/files.js');
 var httpHelpers = require('../utils/http-helpers.js');
 var fiberHelpers = require('../utils/fiber-helpers.js');
+var utils = require('../utils/utils.js');
 
 // Use `METEOR_WAREHOUSE_URLBASE` to override the default warehouse
-// url base. 
+// url base.
 var WAREHOUSE_URLBASE = process.env.METEOR_WAREHOUSE_URLBASE || 'https://warehouse.meteor.com';
 
 var warehouse = exports;
@@ -408,15 +409,6 @@ _.extend(warehouse, {
   },
 
   _platform: function () {
-    // Normalize from Node "os.arch()" to "uname -m".
-    var arch = os.arch();
-    if (arch === "ia32") {
-      arch = "i686";
-    } else if (arch === "x64") {
-      arch = "x86_64";
-    } else {
-      throw new Error("Unsupported architecture " + arch);
-    }
-    return os.type() + "_" + arch;
+    return os.type() + "_" + utils.architecture();
   }
 });

tools/runners/run-mongo.js

@@ -12,7 +12,10 @@ var Console = require('../console/console.js').Console;
 // Given a Mongo URL, open an interative Mongo shell on this terminal
 // on that database.
 var runMongoShell = function (url) {
-  var mongoPath = files.pathJoin(files.getDevBundle(), 'mongodb', 'bin', 'mongo');
+  const architecture = utils.architecture();
+  var mongoPath = files.pathJoin(
+    files.getDevBundle(), 'mongodb', architecture, 'bin', 'mongo'
+  );
   // XXX mongo URLs are not real URLs (notably, the comma-separation for
   // multiple hosts). We've had a little better luck using the mongodb-uri npm
   // package.
@@ -52,12 +55,13 @@ function spawnMongod(mongodPath, port, dbPath, replSetName) {
     // Use an 8MB oplog rather than 256MB. Uses less space on disk and
     // initializes faster. (Not recommended for production!)
     '--oplogSize', '8',
-    '--replSet', replSetName
+    '--replSet', replSetName,
+    '--noauth'
   ];
 
   // Use mmapv1 on 32bit platforms, as our binary doesn't support WT
-  if (process.platform === "win32"
-      || (process.platform === "linux" && process.arch === "ia32")) {
+  const architecture = utils.architecture();
+  if (architecture === 'i386' || architecture === 'i686') {
     args.push('--storageEngine', 'mmapv1', '--smallfiles');
   } else {
     // The WT journal seems to be at least 300MB, which is just too much
@@ -360,8 +364,10 @@ var launchMongo = function (options) {
   var onExit = options.onExit || function () {};
 
   var noOplog = false;
+  const architecture = utils.architecture();
   var mongod_path = files.pathJoin(
-    files.getDevBundle(), 'mongodb', 'bin', 'mongod');
+    files.getDevBundle(), 'mongodb', architecture, 'bin', 'mongod'
+  );
   var replSetName = 'meteor';
 
   // Automated testing: If this is set, instead of starting mongod, we
@@ -533,12 +539,12 @@ var launchMongo = function (options) {
       // note: don't use "else ifs" in this, because 'data' can have multiple
       // lines
       if (/\[initandlisten\] Did not find local replica set configuration document at startup/.test(data) ||
-          /\[ReplicationExecutor\] Locally stored replica set configuration does not have a valid entry for the current node/.test(data)) {
+          /\[.*\] Locally stored replica set configuration does not have a valid entry for the current node/.test(data)) {
         replSetReadyToBeInitiated = true;
         maybeReadyToTalk();
       }
 
-      if (/ \[initandlisten\] waiting for connections on port/.test(data)) {
+      if (/ \[.*\] waiting for connections on port/.test(data)) {
         listening = true;
         maybeReadyToTalk();
       }

tools/tests/mongo.js

@@ -26,17 +26,17 @@ var testMeteorMongo = function (appDir) {
   s.cd(appDir);
 
   var run = s.run();
-  run.waitSecs(15);
   run.match(appDir);
   run.match('proxy');
+  run.waitSecs(15);
   run.match('Started MongoDB');
 
   var mongoRun = s.run('mongo');
   mongoRun.match('MongoDB shell');
-  mongoRun.match('connecting to: 127.0.0.1');
+  mongoRun.match(/connecting to: (.*)127.0.0.1/);
   // Note: when mongo shell's input is not a tty, there is no prompt.
   mongoRun.write('db.version()\n');
-  mongoRun.match(/3\.2\.\d+/);
+  mongoRun.match(/3\.\d+\.\d+/);
   mongoRun.stop();
 
   run.stop();

tools/utils/utils.js

@@ -691,3 +691,47 @@ exports.sourceMapLength = function (sm) {
          return soFar + (current ? current.length : 0);
        }, 0);
 };
+
+// Find and return the current OS architecture, in "uname -m" format.
+//
+// For Linux and macOS (Darwin) this means first getting the current
+// architecture reported by Node using "os.arch()" (e.g. ia32, x64), then
+// converting it to a "uname -m" matching architecture label (e.g. i686,
+// x86_64).
+//
+// For Windows things are handled differently. Node's "os.arch()" will return
+// "ia32" for both 32-bit and 64-bit versions of Windows (since we're using
+// a 32-bit version of Node on Windows). Instead we'll look for the presence
+// of the PROCESSOR_ARCHITEW6432 environment variable to determine if the
+// Windows architecture is 64-bit, then convert to a "uname -m" matching
+// architecture label (e.g. i386, x86_64).
+exports.architecture = () => {
+  const supportedArchitectures = {
+    Darwin: {
+      x64: 'x86_64',
+    },
+    Linux: {
+      ia32: 'i686',
+      x64: 'x86_64',
+    },
+    Windows_NT: {
+      ia32: process.env.hasOwnProperty('PROCESSOR_ARCHITEW6432')
+              ? 'x86_64'
+              : 'i386',
+      x64: 'x86_64'
+    }
+  };
+
+  const osType = os.type();
+  const osArch = os.arch();
+
+  if (!supportedArchitectures[osType]) {
+    throw new Error(`Unsupported OS ${osType}`);
+  }
+
+  if (!supportedArchitectures[osType][osArch]) {
+    throw new Error(`Unsupported architecture ${osArch}`);
+  }
+
+  return supportedArchitectures[osType][osArch];
+};