update build scripts to use a single prebuilds artifact (DataDog#767)

.circleci/config.yml

@@ -762,10 +762,20 @@ jobs:
     working_directory: ~/dd-trace-js
     resource_class: small
     steps:
+      - checkout
       - attach_workspace:
           at: ~/dd-trace-js
+      - *yarn-versions
+      - *restore-yarn-cache
+      - *yarn-install
+      - *save-yarn-cache
+      - run:
+          name: Create prebuilds archive
+          command: yarn prebuilds
+      - store_artifacts:
+          path: ./prebuilds.tgz
       - store_artifacts:
-          path: ./prebuilds
+          path: ./prebuilds.tgz.sha1
 
   # Browser
 

.gitignore

@@ -109,6 +109,7 @@ out
 versions
 build
 prebuilds
+prebuilds.*
 docs/test.js
 !packages/*/test/**/node_modules
 dist

.npmignore

@@ -9,12 +9,7 @@
 !LICENSE
 !LICENSE-3rdparty.csv
 !README.md
-!addons-darwin-ia32.tgz
-!addons-darwin-x64.tgz
-!addons-linux-ia32.tgz
-!addons-linux-x64.tgz
-!addons-win32-ia32.tgz
-!addons-win32-x64.tgz
+!prebuilds.tgz
 !binding.gyp
 !index.d.ts
 !index.js

LICENSE-3rdparty.csv

@@ -47,6 +47,7 @@ dev,eslint-plugin-standard,MIT,Copyright 2015 Jamund Ferguson
 dev,eventemitter3,MIT,Copyright 2014 Arnout Kazemier
 dev,express,MIT,Copyright 2009-2014 TJ Holowaychuk 2013-2014 Roman Shtylman 2014-2015 Douglas Christopher Wilson
 dev,get-port,MIT,Copyright Sindre Sorhus
+dev,glob,ISC,Copyright Isaac Z. Schlueter and Contributors
 dev,graphql,MIT,Copyright 2015 Facebook Inc.
 dev,karma,MIT,Copyright 2011-2019 Google, Inc.
 dev,karma-browserstack-launcher,MIT,Copyright 2011-2013 Google, Inc.

package.json

@@ -8,6 +8,7 @@
   "scripts": {
     "rebuild": "node-gyp rebuild",
     "prebuild": "node scripts/prebuild.js",
+    "prebuilds": "node scripts/prebuilds.js",
     "bundle": "webpack --display-modules",
     "prepublishOnly": "node scripts/prepublish.js",
     "postpublish": "node scripts/postpublish.js",
@@ -99,6 +100,7 @@
     "eventemitter3": "^3.1.0",
     "express": "^4.16.2",
     "get-port": "^3.2.0",
+    "glob": "^7.1.6",
     "graphql": "0.13.2",
     "karma": "^4.3.0",
     "karma-browserstack-launcher": "^1.5.1",

scripts/prebuilds.js

@@ -0,0 +1,79 @@
+'use strict'
+
+const checksum = require('checksum')
+const fs = require('fs')
+const glob = require('glob')
+const os = require('os')
+const path = require('path')
+const tar = require('tar')
+
+const platforms = [
+  'darwin-ia32',
+  'darwin-x64',
+  'linux-ia32',
+  'linux-x64',
+  'win32-ia32',
+  'win32-x64'
+]
+
+zipPrebuilds()
+extractPrebuilds()
+validatePrebuilds()
+createChecksum()
+copyPrebuilds()
+
+function zipPrebuilds () {
+  tar.create({
+    gzip: true,
+    sync: true,
+    portable: true,
+    strict: true,
+    file: path.join(os.tmpdir(), 'prebuilds.tgz')
+  }, glob.sync('prebuilds/**/*.node'))
+}
+
+function extractPrebuilds () {
+  tar.extract({
+    sync: true,
+    strict: true,
+    file: path.join(os.tmpdir(), 'prebuilds.tgz'),
+    cwd: os.tmpdir()
+  })
+}
+
+function validatePrebuilds () {
+  platforms.forEach(platform => {
+    fs.readdirSync(path.join(os.tmpdir(), 'prebuilds', platform))
+      .filter(file => /^node-\d+\.node$/.test(file))
+      .forEach(file => {
+        const content = fs.readFileSync(path.join('prebuilds', platform, file))
+        const sum = fs.readFileSync(path.join('prebuilds', platform, `${file}.sha1`), 'ascii')
+
+        if (sum !== checksum(content)) {
+          throw new Error(`Invalid checksum for "prebuilds/${platform}/${file}".`)
+        }
+      })
+  })
+}
+
+function createChecksum () {
+  const file = path.join(os.tmpdir(), 'prebuilds.tgz')
+  const sum = checksum(fs.readFileSync(file))
+
+  fs.writeFileSync(`${file}.sha1`, sum)
+}
+
+function copyPrebuilds () {
+  const basename = path.normalize(path.join(__dirname, '..'))
+  const filename = 'prebuilds.tgz'
+
+  fs.copyFileSync(
+    path.join(os.tmpdir(), filename),
+    path.join(basename, filename)
+  )
+
+  fs.copyFileSync(
+    path.join(os.tmpdir(), `${filename}.sha1`),
+    path.join(basename, `${filename}.sha1`)
+  )
+}

scripts/prepublish.js

@@ -5,10 +5,8 @@
 const axios = require('axios')
 const checksum = require('checksum')
 const fs = require('fs')
-const mkdirp = require('mkdirp')
 const os = require('os')
 const path = require('path')
-const tar = require('tar')
 const exec = require('./helpers/exec')
 const title = require('./helpers/title')
 
@@ -29,15 +27,6 @@ const branch = exec.pipe(`git symbolic-ref --short HEAD`)
 
 console.log(branch)
 
-const platforms = [
-  'darwin-ia32',
-  'darwin-x64',
-  'linux-ia32',
-  'linux-x64',
-  'win32-ia32',
-  'win32-x64'
-]
-
 const client = axios.create({
   baseURL: 'https://circleci.com/api/v2/',
   timeout: 5000,
@@ -59,9 +48,6 @@ getPipeline()
   .then(getPrebuildsJob)
   .then(getPrebuildArtifacts)
   .then(downloadArtifacts)
-  .then(zipPrebuilds)
-  .then(copyPrebuilds)
-  .then(extractPrebuilds)
   .then(validatePrebuilds)
   .then(bundle)
   .catch(e => {
@@ -126,7 +112,7 @@ function getPrebuildArtifacts (job) {
   return fetch(`project/github/DataDog/dd-trace-js/${job.job_number}/artifacts`)
     .then(response => {
       const artifacts = response.data.items
-        .filter(artifact => /\/prebuilds\//.test(artifact.url))
+        .filter(artifact => /\/prebuilds\.tgz/.test(artifact.url))
 
       if (artifacts.length === 0) {
         throw new Error(`Missing artifacts in job ${job.job_number}.`)
@@ -139,21 +125,16 @@ function getPrebuildArtifacts (job) {
 function downloadArtifacts (artifacts) {
   const files = artifacts.map(artifact => artifact.url)
 
-  return Promise.all([
-    Promise.all(files.map(downloadArtifact)),
-    Promise.all(files.map(file => downloadArtifact(`${file}.sha1`)))
-  ])
+  return Promise.all(files.map(downloadArtifact))
 }
 
 function downloadArtifact (file) {
   return fetch(file, { responseType: 'stream' })
     .then(response => {
       const parts = file.split('/')
-      const basename = path.join(os.tmpdir(), parts.slice(-3, -1).join(path.sep))
+      const basename = os.tmpdir()
       const filename = parts.slice(-1)[0]
 
-      mkdirp.sync(basename)
-
       return new Promise((resolve, reject) => {
         response.data.pipe(fs.createWriteStream(path.join(basename, filename)))
           .on('finish', () => resolve())
@@ -162,55 +143,14 @@ function downloadArtifact (file) {
     })
 }
 
-function zipPrebuilds () {
-  platforms.forEach(platform => {
-    tar.create({
-      gzip: true,
-      sync: true,
-      portable: true,
-      strict: true,
-      file: path.join(os.tmpdir(), `addons-${platform}.tgz`),
-      cwd: os.tmpdir()
-    }, [`prebuilds/${platform}`])
-  })
-}
-
-function copyPrebuilds () {
-  const basename = path.normalize(path.join(__dirname, '..'))
-
-  platforms
-    .map(platform => `addons-${platform}.tgz`)
-    .forEach(filename => {
-      fs.copyFileSync(
-        path.join(os.tmpdir(), filename),
-        path.join(basename, filename)
-      )
-    })
-}
-
-function extractPrebuilds () {
-  platforms.forEach(platform => {
-    tar.extract({
-      sync: true,
-      strict: true,
-      file: `addons-${platform}.tgz`
-    })
-  })
-}
-
 function validatePrebuilds () {
-  platforms.forEach(platform => {
-    fs.readdirSync(path.join('prebuilds', platform))
-      .filter(file => /^node-\d+\.node$/.test(file))
-      .forEach(file => {
-        const content = fs.readFileSync(path.join('prebuilds', platform, file))
-        const sum = fs.readFileSync(path.join('prebuilds', platform, `${file}.sha1`), 'ascii')
-
-        if (sum !== checksum(content)) {
-          throw new Error(`Invalid checksum for "prebuilds/${platform}/${file}".`)
-        }
-      })
-  })
+  const file = path.join(os.tmpdir(), 'prebuilds.tgz')
+  const content = fs.readFileSync(file)
+  const sum = fs.readFileSync(path.join(`${file}.sha1`), 'ascii')
+
+  if (sum !== checksum(content)) {
+    throw new Error('Invalid checksum for "prebuilds.tgz".')
+  }
 }
 
 function bundle () {