feat: use electron clang to build native modules (#431)

* feat: use electron clang to build native modules

* fix: ensure the macosSDK is used as the sysroot on macOS

* fix: restore environment variables properly on windows

* fix: pass /p VS flags to node-gyp to use clang-cl on windows

README.md

@@ -135,6 +135,7 @@ import rebuild from 'electron-rebuild';
 //     headerURL (optional) - Default: https://www.electronjs.org/headers - The URL to download Electron header files from
 //     types (optional) - Default: ['prod', 'optional'] - The types of modules to rebuild
 //     mode (optional) - The rebuild mode, either 'sequential' or 'parallel' - Default varies per platform (probably shouldn't mess with this one)
+//     useElectronClang (optional) - Whether to use the clang executable that Electron used when building.  This will guaruntee compiler compatibility
 
 // Returns a Promise indicating whether the operation succeeded or not
 ```

package.json

@@ -42,9 +42,11 @@
     "debug": "^4.1.1",
     "detect-libc": "^1.0.3",
     "fs-extra": "^9.0.1",
+    "got": "^11.7.0",
     "node-abi": "^2.19.1",
     "node-gyp": "^7.1.0",
     "ora": "^5.1.0",
+    "tar": "^6.0.5",
     "yargs": "^16.0.0"
   },
   "devDependencies": {
@@ -56,6 +58,7 @@
     "@types/fs-extra": "^9.0.1",
     "@types/mocha": "^8.0.3",
     "@types/node": "^14.6.0",
+    "@types/tar": "^4.0.3",
     "@types/yargs": "^15.0.5",
     "@typescript-eslint/eslint-plugin": "^4.0.1",
     "@typescript-eslint/parser": "^4.0.1",

src/clang-fetcher.ts

@@ -0,0 +1,135 @@
+import * as cp from 'child_process';
+import * as debug from 'debug';
+import * as fs from 'fs-extra';
+import got from 'got';
+import * as path from 'path';
+import * as tar from 'tar';
+import * as zlib from 'zlib';
+import { ELECTRON_GYP_DIR } from './constants';
+
+const d = debug('electron-rebuild');
+
+function sleep(n: number) {
+  return new Promise(r => setTimeout(r, n));
+}
+
+async function fetch(url: string, responseType: 'buffer' | 'text', retries = 3): Promise<string | Buffer> {
+  if (retries === 0) throw new Error('Failed to fetch a clang resource, run with DEBUG=electron-rebuild for more information');
+  d('downloading:', url);
+  try {
+    const response = await got.get(url, {
+      responseType,
+    });
+    if (response.statusCode !== 200) {
+      d('got bad status code:', response.statusCode);
+      await sleep(2000);
+      return fetch(url, responseType, retries - 1);
+    }
+    d('response came back OK');
+    return response.body as any;
+  } catch (err) {
+    d('request failed for some reason', err);
+    await sleep(2000);
+    return fetch(url, responseType, retries - 1);
+  }
+}
+
+const CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang';
+
+function getPlatformUrlPrefix(hostOS: string) {
+  const prefixMap = {
+      'linux': 'Linux_x64',
+      'darwin': 'Mac',
+      'win32': 'Win',
+  }
+  return CDS_URL + '/' + prefixMap[hostOS] + '/'
+}
+
+function getClangDownloadURL(packageFile: string, packageVersion: string, hostOS: string) {
+  const cdsFile = `${packageFile}-${packageVersion}.tgz`;
+  return getPlatformUrlPrefix(hostOS) + cdsFile;
+}
+
+function getSDKRoot(): string {
+  if (process.env.SDKROOT) return process.env.SDKROOT;
+  const output = cp.execFileSync('xcrun', ['--sdk', 'macosx', '--show-sdk-path']);
+  return output.toString().trim();
+}
+
+export function getClangEnvironmentVars(electronVersion: string) {
+  const clangDir = path.resolve(ELECTRON_GYP_DIR, `${electronVersion}-clang`, 'bin');
+  const clangArgs: string[] = [];
+  if (process.platform === 'darwin') {
+    clangArgs.push('-isysroot', getSDKRoot());
+  }
+
+  const gypArgs = [];
+  if (process.platform === 'win32') {
+    console.log(fs.readdirSync(clangDir));
+    gypArgs.push(`/p:CLToolExe=clang-cl.exe`, `/p:CLToolPath=${clangDir}`);
+  }
+
+  return {
+    env: {
+      CC: `"${path.resolve(clangDir, 'clang')}" ${clangArgs.join(' ')}`,
+      CXX: `"${path.resolve(clangDir, 'clang++')}" ${clangArgs.join(' ')}`,
+    },
+    args: gypArgs,
+  }
+}
+
+function clangVersionFromRevision(update: string): string | null {
+  const regex = /CLANG_REVISION = '([^']+)'\nCLANG_SUB_REVISION = (\d+)\n/g;
+  const clangVersionMatch = regex.exec(update);
+  if (!clangVersionMatch) return null;
+  const [,clangVersion, clangSubRevision] = clangVersionMatch;
+  return `${clangVersion}-${clangSubRevision}`;
+}
+
+function clangVersionFromSVN(update: string): string | null {
+  const regex = /CLANG_REVISION = '([^']+)'\nCLANG_SVN_REVISION = '([^']+)'\nCLANG_SUB_REVISION = (\d+)\n/g;
+  const clangVersionMatch = regex.exec(update);
+  if (!clangVersionMatch) return null;
+  const [,clangVersion, clangSvn, clangSubRevision] = clangVersionMatch;
+  return `${clangSvn}-${clangVersion.substr(0, 8)}-${clangSubRevision}`;
+}
+
+export async function downloadClangVersion(electronVersion: string) {
+  d('fetching clang for Electron:', electronVersion);
+  const clangDirPath = path.resolve(ELECTRON_GYP_DIR, `${electronVersion}-clang`);
+  if (await fs.pathExists(path.resolve(clangDirPath, 'bin', 'clang'))) return;
+  if (!await fs.pathExists(ELECTRON_GYP_DIR)) await fs.mkdirp(ELECTRON_GYP_DIR);
+
+  const electronDeps = await fetch(`https://raw.githubusercontent.com/electron/electron/v${electronVersion}/DEPS`, 'text');
+  const chromiumRevisionExtractor = /'chromium_version':\n\s+'([^']+)/g;
+  const chromiumRevisionMatch = chromiumRevisionExtractor.exec(electronDeps as string);
+  if (!chromiumRevisionMatch) throw new Error('Failed to determine Chromium revision for given Electron version');
+  const chromiumRevision = chromiumRevisionMatch[1];
+  d('fetching clang for Chromium:', chromiumRevision)
+
+  const base64ClangUpdate = await fetch(`https://chromium.googlesource.com/chromium/src.git/+/${chromiumRevision}/tools/clang/scripts/update.py?format=TEXT`, 'text');
+  const clangUpdate = Buffer.from(base64ClangUpdate as string, 'base64').toString('utf8');
+
+  const clangVersionString = clangVersionFromRevision(clangUpdate) || clangVersionFromSVN(clangUpdate);
+  if (!clangVersionString) throw new Error('Failed to determine Clang revision from Electron version');
+  d('fetching clang:', clangVersionString);
+
+  const clangDownloadURL = getClangDownloadURL('clang', clangVersionString, process.platform);
+
+  const contents = await fetch(clangDownloadURL, 'buffer');
+  d('deflating clang');
+  zlib.deflateSync(contents);
+  const tarPath = path.resolve(ELECTRON_GYP_DIR, `${electronVersion}-clang.tar`);
+  if (await fs.pathExists(tarPath)) await fs.remove(tarPath)
+  await fs.writeFile(tarPath, Buffer.from(contents));
+  await fs.mkdirp(clangDirPath);
+  d('tar running on clang');
+  await tar.x({
+    file: tarPath,
+    cwd: clangDirPath,
+  });
+  await fs.remove(tarPath);
+  d('cleaning up clang tar file');
+}
+
+if (process.mainModule === module) downloadClangVersion('11.0.0-beta.1')

src/constants.ts

@@ -0,0 +1,4 @@
+import * as os from 'os';
+import * as path from 'path';
+
+export const ELECTRON_GYP_DIR = path.resolve(os.homedir(), '.electron-gyp');

src/module-rebuilder.ts

@@ -2,13 +2,14 @@ import * as debug from 'debug';
 import * as detectLibc from 'detect-libc';
 import * as fs from 'fs-extra';
 import * as NodeGyp from 'node-gyp';
-import * as os from 'os';
 import * as path from 'path';
 import { cacheModuleState } from './cache';
 import { promisify } from 'util';
 import { readPackageJson } from './read-package-json';
 import { Rebuilder } from './rebuild';
 import { spawn } from '@malept/cross-spawn-promise';
+import { ELECTRON_GYP_DIR } from './constants';
+import { downloadClangVersion, getClangEnvironmentVars } from './clang-fetcher';
 
 const d = debug('electron-rebuild');
 
@@ -74,19 +75,24 @@ export class ModuleRebuilder {
     return false;
   }
 
-  async buildNodeGypArgs(): Promise<string[]> {
+  async buildNodeGypArgs(prefixedArgs: string[]): Promise<string[]> {
     const args = [
       'node',
       'node-gyp',
       'rebuild',
+      ...prefixedArgs,
       `--runtime=electron`,
       `--target=${this.rebuilder.electronVersion}`,
       `--arch=${this.rebuilder.arch}`,
       `--dist-url=${this.rebuilder.headerURL}`,
       '--build-from-source',
-      `--devdir="${path.resolve(os.homedir(), '.electron-gyp')}"`
+      `--devdir="${ELECTRON_GYP_DIR}"`
     ];
 
+    if (process.env.DEBUG) {
+      args.push('--verbose');
+    }
+
     if (this.rebuilder.debug) {
       args.push('--debug');
     }
@@ -159,6 +165,24 @@ export class ModuleRebuilder {
     return fs.pathExists(path.resolve(this.modulePath, 'prebuilds', `${process.platform}-${this.rebuilder.arch}`, `electron-${this.rebuilder.ABI}.node`))
   }
 
+  private restoreEnv(env: any) {
+    const gotKeys = new Set<string>(Object.keys(process.env));
+    const expectedKeys = new Set<string>(Object.keys(env));
+
+    for (const key of Object.keys(process.env)) {
+      if (!expectedKeys.has(key)) {
+        delete process.env[key];
+      } else if (env[key] !== process.env[key]) {
+        process.env[key] = env[key];
+      }
+    }
+    for (const key of Object.keys(env)) {
+      if (!gotKeys.has(key)) {
+        process.env[key] = env[key];
+      }
+    }
+  }
+
   async rebuildNodeGypModule(cacheKey: string): Promise<void> {
     if (this.modulePath.includes(' ')) {
       console.error('Attempting to build a module with a space in the path');
@@ -167,7 +191,18 @@ export class ModuleRebuilder {
       // throw new Error(`node-gyp does not support building modules with spaces in their path, tried to build: ${modulePath}`);
     }
 
-    const nodeGypArgs = await this.buildNodeGypArgs();
+    let env: any;
+    const extraNodeGypArgs: string[] = [];
+
+    if (this.rebuilder.useElectronClang) {
+      env = { ...process.env };
+      await downloadClangVersion(this.rebuilder.electronVersion);
+      const { env: clangEnv, args: clangArgs } = getClangEnvironmentVars(this.rebuilder.electronVersion);
+      Object.assign(process.env, clangEnv);
+      extraNodeGypArgs.push(...clangArgs);
+    }
+
+    const nodeGypArgs = await this.buildNodeGypArgs(extraNodeGypArgs);
     d('rebuilding', this.moduleName, 'with args', nodeGypArgs);
 
     const nodeGyp = NodeGyp();
@@ -177,6 +212,17 @@ export class ModuleRebuilder {
     try {
       process.chdir(this.modulePath);
       while (command) {
+        if (command.name === 'configure') {
+          command.args = command.args.filter((arg: string) => !extraNodeGypArgs.includes(arg));
+        } else if (command.name === 'build' && process.platform === 'win32') {
+          // This is disgusting but it prevents node-gyp from destroying our MSBuild arguments
+          command.args.map = (fn: (arg: string) => string) => {
+            return Array.prototype.map.call(command.args, (arg: string) => {
+              if (arg.startsWith('/p:')) return arg;
+              return fn(arg);
+            });
+          }
+        }
         await promisify(nodeGyp.commands[command.name])(command.args);
         command = nodeGyp.todo.shift();
       }
@@ -191,7 +237,11 @@ export class ModuleRebuilder {
     d('built:', this.moduleName);
     await this.writeMetadata();
     await this.replaceExistingNativeModule();
-    await this.cacheModuleState(cacheKey)
+    await this.cacheModuleState(cacheKey);
+
+    if (this.rebuilder.useElectronClang) {
+      this.restoreEnv(env);
+    }
   }
 
   async rebuildPrebuildModule(cacheKey: string): Promise<boolean> {

src/rebuild.ts

@@ -25,6 +25,7 @@ export interface RebuildOptions {
   mode?: RebuildMode;
   debug?: boolean;
   useCache?: boolean;
+  useElectronClang?: boolean;
   cachePath?: string;
   prebuildTagPrefix?: string;
   projectRootPath?: string;
@@ -68,6 +69,7 @@ export class Rebuilder {
   public prebuildTagPrefix: string;
   public projectRootPath?: string;
   public msvsVersion?: string;
+  public useElectronClang: boolean;
 
   constructor(options: RebuilderOptions) {
     this.lifecycle = options.lifecycle;
@@ -82,6 +84,7 @@ export class Rebuilder {
     this.mode = options.mode || defaultMode;
     this.debug = options.debug || false;
     this.useCache = options.useCache || false;
+    this.useElectronClang = options.useElectronClang || false;
     this.cachePath = options.cachePath || path.resolve(os.homedir(), '.electron-rebuild-cache');
     this.prebuildTagPrefix = options.prebuildTagPrefix || 'v';
     this.msvsVersion = process.env.GYP_MSVS_VERSION;

test/rebuild.ts

@@ -201,4 +201,23 @@ describe('rebuilder', () => {
       await expectNativeModuleToNotBeRebuilt(testModulePath, 'ffi-napi');
     });
   });
+
+  describe('useElectronClang rebuild', function() {
+    this.timeout(10 * 60 * 1000);
+
+    before(resetTestModule);
+    after(cleanupTestModule);
+
+    it('should have rebuilt ffi-napi module using clang mode', async () => {
+      await rebuild({
+        buildPath: testModulePath,
+        electronVersion: testElectronVersion,
+        arch: process.arch,
+        onlyModules: ['ffi-napi'],
+        force: true,
+        useElectronClang: true
+      });
+      await expectNativeModuleToBeRebuilt(testModulePath, 'ffi-napi');
+    });
+  });
 });