Build stable and experimental with same command (#20573)

acdlite · web-flow · commit eb0fb382308f · 2021-01-12T09:32:32.000-08:00
The goal is to simplify our CI pipeline so that all configurations
are built and tested in a single workflow.

As a first step, this adds a new build script entry point that builds
both the experimental and stable release channels into a single
artifacts directory.

The script works by wrapping the existing build script (which only
builds a single release channel at a time), then post-processing the
results to match the desired filesystem layout. A future version of the
build script would output the files directly without post-processing.

Because many parts of our infra depend on the existing layout of the
build artifacts directory, I have left the old workflows untouched.
We can incremental migrate to the new layout, then delete the old
workflows after we've finished.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -293,6 +293,45 @@ jobs:
             - dist
             - sizes/*.json
 
+  yarn_build_combined:
+    docker: *docker
+    environment: *environment
+    parallelism: 40
+    steps:
+      - checkout
+      - run: yarn workspaces info | head -n -1 > workspace_info.txt
+      - *restore_node_modules
+      - run:
+          command: |
+            ./scripts/circleci/add_build_info_json.sh
+            ./scripts/circleci/update_package_versions.sh
+            yarn build-combined
+      - persist_to_workspace:
+          root: build2
+          paths:
+            - facebook-www
+            - facebook-react-native
+            - facebook-relay
+            - oss-stable
+            - oss-experimental
+            - react-native
+            - dist
+            - sizes/*.json
+
+  process_artifacts_combined:
+    docker: *docker
+    environment: *environment
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build2
+      - run: yarn workspaces info | head -n -1 > workspace_info.txt
+      - *restore_node_modules
+        # Compress build directory into a single tarball for easy download
+      - run: tar -zcvf ./build2.tgz ./build2
+      - store_artifacts:
+          path: ./build2.tgz
+
   build_devtools_and_process_artifacts:
     docker: *docker
     environment: *environment
@@ -611,6 +650,17 @@ workflows:
               only:
                 - master
 
+  # New workflow that will replace "stable" and "experimental"
+  combined:
+    jobs:
+      - setup
+      - yarn_build_combined:
+          requires:
+            - setup
+      - process_artifacts_combined:
+          requires:
+            - yarn_build_combined
+
   fuzz_tests:
     triggers:
       - schedule:
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@ scripts/flow/*/.flowconfig
 _SpecRunner.html
 __benchmarks__
 build/
+build2/
 remote-repo/
 coverage/
 .module-cache
diff --git a/package.json b/package.json
@@ -106,6 +106,7 @@
   },
   "scripts": {
     "build": "node ./scripts/rollup/build.js",
+    "build-combined": "node ./scripts/rollup/build-all-release-channels.js",
     "build-for-devtools": "cross-env RELEASE_CHANNEL=experimental yarn build react/index,react-dom,react-is,react-debug-tools,scheduler,react-test-renderer,react-refresh",
     "build-for-devtools-dev": "yarn build-for-devtools --type=NODE_DEV",
     "build-for-devtools-prod": "yarn build-for-devtools --type=NODE_PROD",
diff --git a/scripts/rollup/build-all-release-channels.js b/scripts/rollup/build-all-release-channels.js
@@ -0,0 +1,123 @@
+'use strict';
+
+/* eslint-disable no-for-of-loops/no-for-of-loops */
+
+const fs = require('fs');
+const {spawnSync} = require('child_process');
+const tmp = require('tmp');
+
+// Runs the build script for both stable and experimental release channels,
+// by configuring an environment variable.
+
+if (process.env.CIRCLE_NODE_TOTAL) {
+  // In CI, we use multiple concurrent processes. Allocate half the processes to
+  // build the stable channel, and the other half for experimental. Override
+  // the environment variables to "trick" the underlying build script.
+  const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
+  const halfTotal = Math.floor(total / 2);
+  const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
+  if (index < halfTotal) {
+    const nodeTotal = halfTotal;
+    const nodeIndex = index;
+    buildForChannel('stable', nodeTotal, nodeIndex);
+    processStable('./build');
+  } else {
+    const nodeTotal = total - halfTotal;
+    const nodeIndex = index - halfTotal;
+    buildForChannel('experimental', nodeTotal, nodeIndex);
+    processExperimental('./build');
+  }
+
+  // TODO: Currently storing artifacts as `./build2` so that it doesn't conflict
+  // with old build job. Remove once we migrate rest of build/test pipeline.
+  fs.renameSync('./build', './build2');
+} else {
+  // Running locally, no concurrency. Move each channel's build artifacts into
+  // a temporary directory so that they don't conflict.
+  buildForChannel('stable', '', '');
+  const stableDir = tmp.dirSync().name;
+  fs.renameSync('./build', stableDir);
+  processStable(stableDir);
+
+  buildForChannel('experimental', '', '');
+  const experimentalDir = tmp.dirSync().name;
+  fs.renameSync('./build', experimentalDir);
+  processExperimental(experimentalDir);
+
+  // Then merge the experimental folder into the stable one. processExperimental
+  // will have already removed conflicting files.
+  //
+  // In CI, merging is handled automatically by CircleCI's workspace feature.
+  spawnSync('rsync', ['-ar', experimentalDir + '/', stableDir + '/']);
+
+  // Now restore the combined directory back to its original name
+  // TODO: Currently storing artifacts as `./build2` so that it doesn't conflict
+  // with old build job. Remove once we migrate rest of build/test pipeline.
+  fs.renameSync(stableDir, './build2');
+}
+
+function buildForChannel(channel, nodeTotal, nodeIndex) {
+  spawnSync('node', ['./scripts/rollup/build.js', ...process.argv.slice(2)], {
+    stdio: ['pipe', process.stdout, process.stderr],
+    env: {
+      ...process.env,
+      RELEASE_CHANNEL: channel,
+      CIRCLE_NODE_TOTAL: nodeTotal,
+      CIRCLE_NODE_INDEX: nodeIndex,
+    },
+  });
+}
+
+function processStable(buildDir) {
+  if (fs.existsSync(buildDir + '/node_modules')) {
+    fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-stable');
+  }
+
+  if (fs.existsSync(buildDir + '/facebook-www')) {
+    for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) {
+      const filePath = buildDir + '/facebook-www/' + fileName;
+      const stats = fs.statSync(filePath);
+      if (!stats.isDirectory()) {
+        fs.renameSync(filePath, filePath.replace('.js', '.classic.js'));
+      }
+    }
+  }
+
+  if (fs.existsSync(buildDir + '/sizes')) {
+    fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-stable');
+  }
+}
+
+function processExperimental(buildDir) {
+  if (fs.existsSync(buildDir + '/node_modules')) {
+    fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-experimental');
+  }
+
+  if (fs.existsSync(buildDir + '/facebook-www')) {
+    for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) {
+      const filePath = buildDir + '/facebook-www/' + fileName;
+      const stats = fs.statSync(filePath);
+      if (!stats.isDirectory()) {
+        fs.renameSync(filePath, filePath.replace('.js', '.modern.js'));
+      }
+    }
+  }
+
+  if (fs.existsSync(buildDir + '/sizes')) {
+    fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-experimental');
+  }
+
+  // Delete all other artifacts that weren't handled above. We assume they are
+  // duplicates of the corresponding artifacts in the stable channel. Ideally,
+  // the underlying build script should not have produced these files in the
+  // first place.
+  for (const pathName of fs.readdirSync(buildDir)) {
+    if (
+      pathName !== 'oss-experimental' &&
+      pathName !== 'facebook-www' &&
+      pathName !== 'sizes-experimental'
+    ) {
+      spawnSync('rm', ['-rm', buildDir + '/' + pathName]);
+    }
+  }
+}
