Run integration/firestore tests in Saucelabs (#3319)

Author: hsubox76

config/karma.saucelabs.js

@@ -46,11 +46,22 @@ const browserMap = {
  * Any special options per package.
  */
 const packageConfigs = {
-  messaging: {
+  '@firebase/messaging': {
     // Messaging currently only supports these browsers.
     browsers: ['Chrome_Windows', 'Firefox_Windows', 'Edge_Windows']
   },
-  firestore: {
+  // Firestore unit tests have OOM problems compiling with Babel for IE.
+  // Firestore integration/firestore tests do run on all browsers.
+  '@firebase/firestore': {
+    browsers: [
+      'Chrome_Windows',
+      'Firefox_Windows',
+      'Edge_Windows',
+      'Safari_macOS'
+    ]
+  },
+  // Installations has IE errors related to `idb` library that need to be figured out.
+  '@firebase/installations': {
     browsers: [
       'Chrome_Windows',
       'Firefox_Windows',
@@ -84,12 +95,12 @@ function getSauceLabsBrowsers(packageName) {
  */
 function getPackageLabels() {
   const match = testConfigFile.match(
-    /([a-zA-Z]+)\/([a-zA-Z-]+)\/karma\.conf\.js/
+    /([a-zA-Z]+\/[a-zA-Z-]+)\/karma\.conf\.js/
   );
-  return {
-    type: match[1],
-    name: match[2]
-  };
+  const packagePath = match[1];
+  const root = path.resolve(__dirname, '..');
+  const pkg = require(path.join(root, packagePath, 'package.json'));
+  return pkg.name;
 }
 
 /**
@@ -106,11 +117,8 @@ function getTestFiles() {
 }
 
 function seleniumLauncher(browserName, platform, version) {
-  const { name, type } = getPackageLabels();
-  const testName =
-    type === 'integration'
-      ? `${type}-${name}-${browserName}`
-      : `${name}-${browserName}`;
+  const { name } = getPackageLabels();
+  const testName = `${name}-${browserName}`;
   return {
     base: 'SauceLabs',
     browserName: browserName,
@@ -150,8 +158,8 @@ module.exports = function(config) {
   const sauceLabsBrowsers = getSauceLabsBrowsers(packageName);
 
   const sauceLabsConfig = {
-    tunnelIdentifier: process.env.TRAVIS_JOB_NUMBER + '-' + packageName,
-    build: process.env.TRAVIS_BUILD_NUMBER || argv['buildNumber'],
+    tunnelIdentifier: process.env.GITHUB_RUN_ID + '-' + packageName,
+    build: process.env.GITHUB_RUN_ID || argv['buildNumber'],
     username: process.env.SAUCE_USERNAME,
     accessKey: process.env.SAUCE_ACCESS_KEY,
     startConnect: true,
@@ -210,7 +218,7 @@ module.exports = function(config) {
       maxLogLines: 5,
       suppressErrorSummary: false,
       suppressFailed: false,
-      suppressPassed: false,
+      suppressPassed: true,
       suppressSkipped: true,
       showSpecTiming: true,
       failFast: false

integration/firestore/karma.conf.js

@@ -1,6 +1,6 @@
 /**
  * @license
- * Copyright 2017 Google Inc.
+ * Copyright 2017 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,11 +19,7 @@ const karma = require('karma');
 const path = require('path');
 const karmaBase = require('../../config/karma.base');
 
-const files = [
-  `${path.dirname(require.resolve('firebase'))}/firebase.js`,
-  `${path.dirname(require.resolve('firebase'))}/firebase-firestore.js`,
-  './dist/test-harness.js'
-];
+const files = ['./dist/test-harness.js'];
 
 module.exports = function(config) {
   const karmaConfig = Object.assign({}, karmaBase, {

packages/polyfill/index.ts

@@ -38,3 +38,4 @@ import 'core-js/features/symbol/iterator';
 import 'core-js/features/map';
 import 'core-js/features/set';
 import 'core-js/features/number/is-integer';
+import 'core-js/features/number/is-nan';

scripts/run_saucelabs.js

@@ -20,6 +20,7 @@ const { exists } = require('mz/fs');
 const yargs = require('yargs');
 const glob = require('glob');
 const path = require('path');
+const chalk = require('chalk');
 
 // Check for 'configFiles' flag to run on specified karma.conf.js files instead
 // of on all files.
@@ -37,9 +38,11 @@ const { configFiles } = yargs
 const testFiles = configFiles.length
   ? configFiles
   : glob
-      .sync(`{packages,integration}/*/karma.conf.js`)
-      // Automated tests in integration/firestore are currently disabled.
-      .filter(name => !name.includes('integration/firestore'));
+      .sync(`packages/*/karma.conf.js`)
+      // Skip integration namespace tests, not very useful, introduce errors.
+      .concat('integration/firestore/karma.conf.js')
+      // Exclude database - currently too many failures.
+      .filter(name => !name.includes('packages/database'));
 
 // Get CI build number or generate one if running locally.
 const buildNumber =
@@ -54,22 +57,60 @@ const buildNumber =
  * group.
  */
 async function runTest(testFile) {
+  if (!(await exists(testFile))) {
+    console.error(chalk`{red ERROR: ${testFile} does not exist.}`);
+    return 1;
+  }
   // Run pretest if this dir has a package.json with a pretest script.
   const testFileDir =
     path.resolve(__dirname, '../') + '/' + path.dirname(testFile);
   const pkgPath = testFileDir + '/package.json';
+  let pkgName = testFile;
   if (await exists(pkgPath)) {
     const pkg = require(pkgPath);
+    pkgName = pkg.name;
     if (pkg.scripts.pretest) {
       await spawn('yarn', ['--cwd', testFileDir, 'pretest'], {
         stdio: 'inherit'
       });
     }
   }
-  return runKarma(testFile);
+  if (testFile.includes('integration/firestore')) {
+    console.log(
+      chalk`{blue Generating memory-only build for integration/firestore.}`
+    );
+    await spawn('yarn', ['--cwd', 'integration/firestore', 'build:memory'], {
+      stdio: 'inherit'
+    });
+    console.log(
+      chalk`{blue Running tests on memory-only build for integration/firestore.}`
+    );
+    const exitCode1 = await runKarma(testFile, `${pkgName}-memory`);
+    console.log(
+      chalk`{blue Generating persistence build for integration/firestore.}`
+    );
+    await spawn(
+      'yarn',
+      ['--cwd', 'integration/firestore', 'build:persistence'],
+      { stdio: 'inherit' }
+    );
+    console.log(
+      chalk`{blue Running tests on persistence build for integration/firestore.}`
+    );
+    const exitCode2 = await runKarma(testFile, `${pkgName}-persistence`);
+    return Math.max(exitCode1, exitCode2);
+  } else {
+    return runKarma(testFile, pkgName);
+  }
 }
 
-async function runKarma(testFile) {
+/**
+ * Runs the karma test command for one package.
+ *
+ * @param {string} testFile - path to karma.conf.js file
+ * @param {string} testTag - package label for messages (usually package name)
+ */
+async function runKarma(testFile, testTag) {
   const karmaArgs = [
     'karma',
     'start',
@@ -92,11 +133,11 @@ async function runKarma(testFile) {
 
   return promise
     .then(() => {
-      console.log(`[${testFile}] ******* DONE *******`);
+      console.log(chalk`{green [${testTag}] ******* DONE *******}`);
       return exitCode;
     })
     .catch(err => {
-      console.error(`[${testFile}] ERROR:`, err.message);
+      console.error(chalk`{red [${testTag}] ERROR: ${err.message}}`);
       return exitCode;
     });
 }
@@ -109,19 +150,29 @@ async function runKarma(testFile) {
  * of all child processes.  This allows any failing test to result in a CI
  * build failure for the whole Saucelabs run.
  */
-async function runNextTest(maxExitCode = 0) {
+async function runNextTest(maxExitCode = 0, results = {}) {
   // When test queue is empty, exit with code 0 if no tests failed or
   // 1 if any tests failed.
-  if (!testFiles.length) process.exit(maxExitCode);
+  if (!testFiles.length) {
+    for (const fileName of Object.keys(results)) {
+      if (results[fileName] > 0) {
+        console.log(`FAILED: ${fileName}`);
+      }
+    }
+    process.exit(maxExitCode);
+  }
   const nextFile = testFiles.shift();
   let exitCode;
   try {
     exitCode = await runTest(nextFile);
   } catch (e) {
-    console.error(`[${nextFile}] ERROR:`, e.message);
+    console.error(chalk`{red [${nextFile}] ERROR: ${e.message}}`);
     exitCode = 1;
   }
-  runNextTest(Math.max(exitCode, maxExitCode));
+  runNextTest(Math.max(exitCode, maxExitCode), {
+    ...results,
+    [nextFile]: exitCode
+  });
 }
 
 runNextTest();