Add test suite & two ways to run it (via yarn, via VS Code)

Author: xeger

.gitignore

@@ -1,4 +1,5 @@
 .token
+.vscode-test
 node_modules
 *.vsix
 out

.vscode/launch.json

@@ -5,6 +5,7 @@
 {
 	"version": "0.2.0",
 	"configurations": [
+
 		{
 			"name": "Run Extension",
 			"type": "extensionHost",
@@ -17,5 +18,17 @@
 			],
 			"preLaunchTask": "${defaultBuildTask}"
 		},
+		{
+			"name": "Run Extension Tests",
+			"type": "extensionHost",
+			"request": "launch",
+			"runtimeExecutable": "${execPath}",
+			"args": [
+				"--extensionDevelopmentPath=${workspaceFolder}",
+				"--extensionTestsPath=${workspaceFolder}/out/test/suite/index"
+			],
+			"outFiles": ["${workspaceFolder}/out/test/**/*.js"],
+			"preLaunchTask": "${defaultBuildTask}"
+		}
 	]
 }

package.json

@@ -13,6 +13,7 @@
   "bugs": {
     "url": "https://github.com/ruby-syntax-tree/vscode-syntax-tree/issues"
   },
+  "packageManager": "yarn@1.22.19",
   "engines": {
     "vscode": "^1.66.0"
   },
@@ -76,18 +77,25 @@
     "colors": []
   },
   "scripts": {
+    "clean": "rm -Rf out",
     "compile": "tsc -p ./",
     "package": "vsce package --yarn --githubBranch main",
     "publish": "vsce publish --yarn --githubBranch main",
     "vscode:prepublish": "yarn compile",
+    "test": "node ./out/test/runTest.js",
     "watch": "tsc --watch -p ./"
   },
   "dependencies": {
     "vscode-languageclient": "8.0.2-next.5"
   },
   "devDependencies": {
+    "@types/glob": "^7.1.1",
+    "@types/mocha": "^9.1.1",
     "@types/node": "^18.0.0",
     "@types/vscode": "^1.68.0",
+    "@vscode/test-electron": "^1.6.1",
+    "glob": "^7.1.4",
+    "mocha": "^9.1.1",
     "typescript": "^4.7.4",
     "vsce": "^2.9.2"
   }

src/test/runTest.ts

@@ -0,0 +1,24 @@
+import { runTests } from '@vscode/test-electron';
+import * as path from 'path';
+
+import { USER_DATA_DIR, WORKSPACE_DIR } from './suite/setup';
+
+async function main() {
+  try {
+    // The folder containing the Extension Manifest package.json
+    // Passed to `--extensionDevelopmentPath`
+    const extensionDevelopmentPath = path.resolve(__dirname, '../../');
+
+    // The path to the extension test script
+    // Passed to --extensionTestsPath
+    const extensionTestsPath = path.resolve(__dirname, './suite/index');
+
+    // Download VS Code, unzip it and run the integration test
+    await runTests({ extensionDevelopmentPath, extensionTestsPath, launchArgs: ['--disable-extensions', '--disable-gpu', '--user-data-dir', USER_DATA_DIR, WORKSPACE_DIR] });
+  } catch (err) {
+    console.error('Failed to run tests');
+    process.exit(1);
+  }
+}
+
+main();

src/test/suite/automation.ts

@@ -0,0 +1,71 @@
+import * as assert from 'assert';
+import * as path from 'path';
+import { TextEncoder } from 'util';
+
+import { Range, Uri, commands, window, workspace } from 'vscode';
+
+import { WORKSPACE_DIR } from './setup';
+
+export async function closeAll() {
+  await commands.executeCommand('workbench.action.closeAllEditors');
+
+  await commands.executeCommand('workbench.panel.output.focus');
+  await commands.executeCommand('workbench.action.output.toggleOutput');
+  await showOutputChannel();
+
+  // @see https://stackoverflow.com/questions/44733028/how-to-close-textdocument-in-vs-code
+  const { visibleTextEditors: editors } = window;
+  for (let i = 0; i < editors.length; i++) {
+    const { document } = editors[i];
+    await window.showTextDocument(document, { preview: true, preserveFocus: true });
+    await editors[i].edit(cb => {
+      cb.delete(new Range(0, 0, document.lineCount, 0));
+    });
+    await commands.executeCommand('workbench.action.closeActiveEditor');
+  }
+  await commands.executeCommand('workbench.action.output.toggleOutput');
+
+  // Set the focus back to something neutral
+  await commands.executeCommand('workbench.explorer.fileView.focus');
+}
+
+export async function createEditor(content: string, filename = 'test.rb') {
+  const uri = Uri.file(`${WORKSPACE_DIR}${path.sep}${filename}`);
+  await workspace.fs.writeFile(uri, new TextEncoder().encode(content));
+  await window.showTextDocument(uri);
+  assert.ok(window.activeTextEditor);
+  assert.equal(window.activeTextEditor.document.getText(), content);
+  return window.activeTextEditor;
+}
+
+export function formatDocument() {
+  return commands.executeCommand('editor.action.formatDocument', 'ruby-syntax-tree.vscode-syntax-tree');
+}
+
+export function restart() {
+  return commands.executeCommand('syntaxTree.restart');
+}
+
+export function showOutputChannel(timeout = 15000) {
+  // TODO: make use of await commands.executeCommand('workbench.panel.output.focus'); ??
+  return new Promise<string>((c, e) => {
+    const failSauce = new Error(`No active editor change within ${timeout}ms`)
+    const timer = setTimeout(() => e(failSauce), timeout);
+    const disposable = window.onDidChangeActiveTextEditor(e => {
+      if (e) {
+        clearTimeout(timer);
+        disposable.dispose();
+        c(e.document.getText())
+      }
+    })
+    commands.executeCommand('syntaxTree.showOutputChannel');
+  })
+}
+
+export function start() {
+  return commands.executeCommand('syntaxTree.start');
+}
+
+export function stop() {
+  return commands.executeCommand('syntaxTree.stop');
+}

src/test/suite/index.test.ts

@@ -0,0 +1,54 @@
+import * as assert from 'assert';
+import { before, beforeEach } from 'mocha';
+
+import * as auto from './automation';
+import { TIMEOUT_MS } from './setup';
+
+// can directly import extension if needed
+// import * as myExtension from '../../extension';
+
+const UNFORMATTED = `class Foo; def bar; puts 'baz'; end; end`;
+
+const FORMATTED = `class Foo
+  def bar
+    puts "baz"
+  end
+end
+`;
+
+suite('Syntax Tree', () => {
+  suite('commands', () => {
+    beforeEach(async () => {
+      await auto.stop();
+      await auto.closeAll();
+    });
+
+    test('showOutputChannel', async () => {
+      const text = await auto.showOutputChannel();
+      assert.match(text, /^Starting language server/m);
+    })
+
+    test('start', async () => {
+      await auto.start();
+      const text = await auto.showOutputChannel();
+      // HACK: with no way to clear the channel, this is a flakey assertion
+      assert.match(text, /^Starting language server/m);
+    });
+
+    test('stop', async () => {
+      await auto.start();
+      await auto.stop();
+      const text = await auto.showOutputChannel();
+      // HACK: with no way to clear the channel, this is a flakey assertion
+      assert.match(text, /^Stopping language server/m);
+    })
+  });
+
+  // TODO: figure out re-entrancy issue in CI (prolly two LanguageClient instances at once?)
+  test.skip('Format Document', async () => {
+    await auto.closeAll();
+    const editor = await auto.createEditor(UNFORMATTED);
+    await auto.formatDocument();
+    assert.equal(editor.document.getText(), FORMATTED);
+  });
+});

src/test/suite/index.ts

@@ -0,0 +1,48 @@
+import * as path from 'path';
+import * as Mocha from 'mocha';
+import * as glob from 'glob';
+
+import { TIMEOUT_MS } from './setup';
+
+export function run(): Promise<void> {
+  const mocha = new Mocha({
+    asyncOnly: true,
+    color: true,
+    forbidOnly: !!process.env.CI,
+    slow: TIMEOUT_MS / 4,
+    timeout: TIMEOUT_MS,
+    ui: 'tdd'
+  });
+
+  const testsRoot = path.resolve(__dirname, '..');
+
+  return new Promise((c, e) => {
+    glob('**/**.test.js', { cwd: testsRoot }, (err, files) => {
+      if (err) {
+        return e(err);
+      }
+
+      // Add files to the test suite
+      files.forEach(f => mocha.addFile(path.resolve(testsRoot, f)));
+
+      try {
+        // Run the mocha test
+        mocha.run(failures => {
+          if (failures > 0) {
+            // Let the cameras roll for a bit & make sure we capture the error
+            if (process.env.CI) {
+              setTimeout(() => e(new Error(`${failures} tests failed; pausing for dramatic effect.`)), 3000);
+            } else {
+              e(new Error(`${failures} tests failed.`));
+            }
+          } else {
+            c();
+          }
+        });
+      } catch (err) {
+        console.error(err);
+        e(err);
+      }
+    });
+  });
+}

src/test/suite/setup.ts

@@ -0,0 +1,25 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+// Share scratch dir between runner and Code process via clever env trick
+const inheritedScratchDir = process.env.RUBY_SYNTAX_TREE_TEST_SCRATCH_DIR;
+const SCRATCH_DIR = inheritedScratchDir || fs.mkdtempSync(`${os.tmpdir()}${path.sep}vscode-syntax-tree-`);
+process.env.RUBY_SYNTAX_TREE_TEST_SCRATCH_DIR = SCRATCH_DIR;
+
+/// How long to wait for each test case before marking it as a failure.
+/// Outside of CI, give the dev 5 minutes to play with debuggers, etc!
+export const TIMEOUT_MS = process.env.CI ? 30000 : 300000;
+
+/// Holds profile, settings, etc - to give us a clean slate every time
+/// & avoid polluting developer's real profile
+export const USER_DATA_DIR = path.join(SCRATCH_DIR, 'user-data');
+
+/// Holds text documents that we author during tests.
+export const WORKSPACE_DIR = path.join(SCRATCH_DIR, 'workspace');
+
+if (!inheritedScratchDir) { // We're the parent; adulting is hard!
+  fs.mkdirSync(USER_DATA_DIR);
+  fs.mkdirSync(WORKSPACE_DIR);
+  console.log('Scratch folder:', SCRATCH_DIR);
+}