May 18, 2020 · May 16, 2020 · May 17, 2020 · May 17, 2020 · May 17, 2020 · May 17, 2020
diff --git a/CHANGELOG.md b/CHANGELOG.md
 ## [0.6.1]

 - Replace checkboxes with icons

 ## [0.7.0]

 - Support loading subtasks (#340). Subtasks are a list of tests that need to pass before a task is complete. They can be loaded by:

 1. filtering down to a subset of tests by setting the `step.setup.filter` to a regex pattern that matches the tests you're targeting
 2. setting the `step.setup.subtasks` variable to true

 - Change for the test runner config. Changes are backwards compatible.

 1. `testRunner.path`=> `testRunner.directory`
 2. `testRunner.actions` => `testRunner.setup`
 3. Change command to capture `args` for "TAP" support, and test "filter"ing support. These changes will help lead to specific test suite presets in the future.

 ```json
 {
  "testRunner": {
    "command": "mocha",
    "args": {
      "filter": "--grep",
      "tap": "--reporter=mocha-tap-reporter"
    },
    "directory": ".coderoad",
    "setup": {
      "commits": ["410bd4f"],
      "commands": ["npm install"]
    }
  }
 }
 ```
diff --git a/package.json b/package.json
    "@typescript-eslint/parser": "^2.33.0",
    "chokidar": "^3.4.0",
    "dotenv": "^8.2.0",
    "eslint": "^7.0.0",
    "eslint": "^6.8.0",
    "eslint-config-prettier": "^6.11.0",
    "eslint-plugin-prettier": "^3.1.3",
    "git-url-parse": "^11.1.2",
diff --git a/src/actions/setupActions.ts b/src/actions/setupActions.ts
 interfaceSetupActions{
 actions:TT.StepActions
 send:(action:T.Action)=>void// send messages to client
 path?:string
 dir?:string
 }

 exportconstsetupActions=async({ actions, send,path}:SetupActions):Promise<void>=>{
 exportconstsetupActions=async({ actions, send,dir}:SetupActions):Promise<void>=>{
 if(!actions){
 return
 }

 // 4. run command
 if(!alreadyLoaded){
 awaitrunCommands({commands:commands||[], send,path}).catch(onError)
 awaitrunCommands({commands:commands||[], send,dir}).catch(onError)
 }
 }

diff --git a/src/actions/tutorialConfig.ts b/src/actions/tutorialConfig.ts
 import { DISABLE_RUN_ON_SAVE } from '../environment'

 interface TutorialConfigParams {
 config: TT.TutorialConfig
 data: TT.Tutorial
  alreadyConfigured?: boolean
  onComplete?(): void
 }

 const tutorialConfig = async ({config, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage | void> => {
 const tutorialConfig = async ({data, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage | void> => {
  if (!alreadyConfigured) {
    // setup git, add remote
    const initError: E.ErrorMessage | void = await git.initIfNotExists().catch(
    }

    // verify that internet is connected, remote exists and branch exists
    const remoteConnectError: E.ErrorMessage | void = await git.checkRemoteConnects(config.repo).catch(
    const remoteConnectError: E.ErrorMessage | void = await git.checkRemoteConnects(data.config.repo).catch(
      (error: Error): E.ErrorMessage => ({
        type: 'FailedToConnectToGitRepo',
        message: error.message,
    }

    // TODO if remote not already set
    const coderoadRemoteError: E.ErrorMessage | void = await git.setupCodeRoadRemote(config.repo.uri).catch(
    const coderoadRemoteError: E.ErrorMessage | void = await git.setupCodeRoadRemote(data.config.repo.uri).catch(
      (error: Error): E.ErrorMessage => ({
        type: 'GitRemoteAlreadyExists',
        message: error.message,
    }
  }

  await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER,config.testRunner)
  await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER,data)

  if (!DISABLE_RUN_ON_SAVE) {
    // verify if file test should run based on document saved
diff --git a/src/actions/utils/loadWatchers.ts b/src/actions/utils/loadWatchers.ts
 constnow=+newDate()
 if(!lastFire||lastFire-now>1000){
 vscode.commands.executeCommand(COMMANDS.RUN_TEST,{
 onSuccess:()=>{
 // cleanup watcher on success
 disposeWatcher(watcher)
 callbacks:{
 onSuccess:()=>{
 // cleanup watcher on success
 disposeWatcher(watcher)
 },
 },
 })
 }
diff --git a/src/actions/utils/runCommands.ts b/src/actions/utils/runCommands.ts
 interfaceRunCommands{
 commands:string[]
 send:(action:T.Action)=>void
 path?:string
 dir?:string
 }

 construnCommands=async({ commands, send,path}:RunCommands)=>{
 construnCommands=async({ commands, send,dir}:RunCommands)=>{
 if(!commands.length){
 return
 }
 send({type:'COMMAND_START',payload:{process:{ ...process,status:'RUNNING'}}})
 letresult:{stdout:string;stderr:string}
 try{
 result=awaitexec({ command,path})
 result=awaitexec({ command,dir})
 console.log(result)
 }catch(error){
 console.log(`Test failed:${error.message}`)
diff --git a/src/channel/index.ts b/src/channel/index.ts
            }
          }

          const error: E.ErrorMessage | void = await tutorialConfig({config:data.config }).catch((error: Error) => ({
          const error: E.ErrorMessage | void = await tutorialConfig({ data }).catch((error: Error) => ({
            type: 'UnknownError',
            message: `Location: tutorial config.\n\n${error.message}`,
          }))
          if (!tutorialContinue) {
            throw new Error('Invalid tutorial to continue')
          }
          const continueConfig: TT.TutorialConfig = tutorialContinue.config
          await tutorialConfig({
 config: continueConfig,
 data: tutorialContinue,
            alreadyConfigured: true,
          })
          // update the current stepId on startup
        await vscode.commands.executeCommand(COMMANDS.SET_CURRENT_POSITION, action.payload.position)
        await solutionActions({ actions: action.payload.actions, send: this.send })
        // run test following solution to update position
        vscode.commands.executeCommand(COMMANDS.RUN_TEST)
        vscode.commands.executeCommand(COMMANDS.RUN_TEST, { subtasks: true })
        return
      case 'EDITOR_SYNC_PROGRESS':
        // update progress when a level is deemed complete in the client
        await showOutput(channel)
        return
      case 'EDITOR_RUN_TEST':
        vscode.commands.executeCommand(COMMANDS.RUN_TEST)
        vscode.commands.executeCommand(COMMANDS.RUN_TEST, action?.payload)
        return
      default:
        logger(`No match for action type: ${actionType}`)
diff --git a/src/editor/commands.ts b/src/editor/commands.ts
      // setup 1x1 horizontal layout
      webview.createOrShow()
    },
    [COMMANDS.CONFIG_TEST_RUNNER]: async (config: TT.TutorialTestRunnerConfig) => {
      if (config.actions) {
    [COMMANDS.CONFIG_TEST_RUNNER]: async (data: TT.Tutorial) => {
      const testRunnerConfig = data.config.testRunner
      const setup = testRunnerConfig.setup || testRunnerConfig.actions // TODO: deprecate and remove config.actions
      if (setup) {
        // setup tutorial test runner commits
        // assumes git already exists
        await setupActions({ actions: config.actions, send: webview.send, path: config.path })
        await setupActions({
          actions: setup,
          send: webview.send,
          dir: testRunnerConfig.directory || testRunnerConfig.path,
        }) // TODO: deprecate and remove config.path
      }
      testRunner = createTestRunner(config, {
      testRunner = createTestRunner(data, {
        onSuccess: (position: T.Position) => {
          logger('test pass position', position)
          // send test pass message back to client
          // send test run message back to client
          webview.send({ type: 'TEST_RUNNING', payload: { position } })
        },
        onLoadSubtasks: ({ summary }) => {
          webview.send({ type: 'LOAD_TEST_SUBTASKS', payload: { summary } })
        },
      })
    },
    [COMMANDS.SET_CURRENT_POSITION]: (position: T.Position) => {
      // set from last setup stepAction
      currentPosition = position
    },
    [COMMANDS.RUN_TEST]: (callback?: { onSuccess: () => void }) => {
    [COMMANDS.RUN_TEST]: ({
      subtasks,
      callbacks,
    }: { subtasks?: boolean; callbacks?: { onSuccess: () => void } } = {}) => {
      logger('run test current', currentPosition)
      // use stepId from client, or last set stepId
      // const position: T.Position = {
      //   ...current,
      //   stepId: current && current.position.stepId?.length ? current.position.stepId : currentPosition.stepId,
      // }
      testRunner(currentPosition, callback?.onSuccess)
      logger('currentPosition', currentPosition)
      testRunner({ position: currentPosition, onSuccess: callbacks?.onSuccess, subtasks })
    },
  }
 }
diff --git a/src/services/node/index.ts b/src/services/node/index.ts

 interfaceExecParams{
 command:string
 path?:string
 dir?:string
 }

 exportconstexec=(params:ExecParams):Promise<{stdout:string;stderr:string}>|never=>{
 constcwd=join(WORKSPACE_ROOT,params.path||'')
 constcwd=join(WORKSPACE_ROOT,params.dir||'')
 returnasyncExec(params.command,{ cwd})
 }

diff --git a/src/services/testRunner/index.ts b/src/services/testRunner/index.ts
 import * as TT from 'typings/tutorial'
 import { exec } from '../node'
 import logger from '../logger'
 import parser from './parser'
 import parser, { ParserOutput } from './parser'
 import { debounce, throttle } from './throttle'
 import onError from '../sentry/onError'
 import { clearOutput, addOutput } from './output'
  onFail(position: T.Position, failSummary: T.TestFail): void
  onRun(position: T.Position): void
  onError(position: T.Position): void
  onLoadSubtasks({ summary }: { summary: { [testName: string]: boolean } }): void
 }

 const failChannelName = 'CodeRoad (Tests)'
 const logChannelName = 'CodeRoad (Logs)'

 const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callbacks) => {
  return async (position: T.Position, onSuccess?: () => void): Promise<void> => {
    logger('createTestRunner', position)
 interface TestRunnerParams {
  position: T.Position
  subtasks?: boolean
  onSuccess?: () => void
 }

 const createTestRunner = (data: TT.Tutorial, callbacks: Callbacks) => {
  const testRunnerConfig = data.config.testRunner
  const testRunnerFilterArg = testRunnerConfig.args?.filter
  return async ({ position, onSuccess, subtasks }: TestRunnerParams): Promise<void> => {
    const startTime = throttle()
    // throttle time early
    if (!startTime) {
    logger('------------------- RUN TEST -------------------')

    // flag as running
    callbacks.onRun(position)
    if (!subtasks) {
      callbacks.onRun(position)
    }

    let result: { stdout: string | undefined; stderr: string | undefined }
    try {
      result = await exec({ command: config.command, path: config.path })
      let command = testRunnerConfig.args
        ? `${testRunnerConfig.command} ${testRunnerConfig?.args.tap}`
        : testRunnerConfig.command // TODO: enforce TAP

      // filter tests if requested
      if (testRunnerFilterArg) {
        // get tutorial step from position
        // check the step actions for specific command
        // NOTE: cannot just pass in step actions as the test can be called by:
        // - onEditorSave, onWatcher, onSolution, onRunTest, onSubTask
        const levels = data.levels
        const level = levels.find((l) => l.id === position.levelId)
        const step = level?.steps.find((s) => s.id === position.stepId)
        const testFilter = step?.setup?.filter
        if (testFilter) {
          // append filter commands
          command = [command, testRunnerFilterArg, testFilter].join(' ')
        } else {
          throw new Error('Test Runner filter not configured')
        }
      }
      logger('COMMAND', command)
      result = await exec({ command, dir: testRunnerConfig.directory || testRunnerConfig.path }) // TODO: remove config.path later
    } catch (err) {
      result = { stdout: err.stdout, stderr: err.stack }
    }

    const { stdout, stderr } = result

    const tap = parser(stdout || '')
    const tap: ParserOutput = parser(stdout || '')

    if (subtasks) {
      callbacks.onLoadSubtasks({ summary: tap.summary })
      // exit early
      return
    }

    addOutput({ channel: logChannelName, text: tap.logs.join('\n'), show: false })

        const failSummary = {
          title: firstFail.message || 'Test Failed',
          description: firstFail.details || 'Unknown error',
          summary: tap.summary,
        }
        callbacks.onFail(position, failSummary)
        const output = formatFailOutput(tap)
    // PASS
    if (tap.ok) {
      clearOutput(failChannelName)

      callbacks.onSuccess(position)

      if (onSuccess) {
        onSuccess()
      }
diff --git a/src/services/testRunner/parser.test.ts b/src/services/testRunner/parser.test.ts
 1..1
 ok 1 - Should pass
 `
 expect(parser(example)).toEqual({ok:true,passed:[{message:'Should pass'}],failed:[],logs:[]})
 expect(parser(example)).toEqual({
 ok:true,
 passed:[{message:'Should pass'}],
 failed:[],
 logs:[],
 summary:{'Should pass':true},
 })
 })
 test('should detect multiple successes',()=>{
 constexample=`
 passed:[{message:'Should pass'},{message:'Should also pass'}],
 failed:[],
 logs:[],
 summary:{
 'Should pass':true,
 'Should also pass':true,
 },
 })
 })
 test('should detect failure if no tests passed',()=>{
 },
 ],
 logs:['log 1','log 2'],
 summary:{
 'package.json should have "express" installed':true,
 'server should log "Hello World"':false,
 },
 })
 })
 })
Original file line number	Diff line number	Diff line change
Expand Up		@@ -125,3 +125,33 @@ CODEROAD_TUTORIAL_URL='path/to/tutorial_config_file.json' // will load directly
		## [0.6.1]

		- Replace checkboxes with icons

		## [0.7.0]

		- Support loading subtasks (#340). Subtasks are a list of tests that need to pass before a task is complete. They can be loaded by:

		1. filtering down to a subset of tests by setting the `step.setup.filter` to a regex pattern that matches the tests you're targeting
		2. setting the `step.setup.subtasks` variable to true

		- Change for the test runner config. Changes are backwards compatible.

		1. `testRunner.path`=> `testRunner.directory`
		2. `testRunner.actions` => `testRunner.setup`
		3. Change command to capture `args` for "TAP" support, and test "filter"ing support. These changes will help lead to specific test suite presets in the future.

		```json
		{
		"testRunner": {
		"command": "mocha",
		"args": {
		"filter": "--grep",
		"tap": "--reporter=mocha-tap-reporter"
		},
		"directory": ".coderoad",
		"setup": {
		"commits": ["410bd4f"],
		"commands": ["npm install"]
		}
		}
		}
		```
Original file line number	Diff line number	Diff line change
Expand Up		@@ -45,7 +45,7 @@
		"@typescript-eslint/parser": "^2.33.0",
		"chokidar": "^3.4.0",
		"dotenv": "^8.2.0",
		"eslint": "^7.0.0",
		"eslint": "^6.8.0",
		"eslint-config-prettier": "^6.11.0",
		"eslint-plugin-prettier": "^3.1.3",
		"git-url-parse": "^11.1.2",
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,10 +10,10 @@ import logger from '../services/logger'
		interfaceSetupActions{
		actions:TT.StepActions
		send:(action:T.Action)=>void// send messages to client
		path?:string
		dir?:string
		}

		exportconstsetupActions=async({ actions, send,path}:SetupActions):Promise<void>=>{
		exportconstsetupActions=async({ actions, send,dir}:SetupActions):Promise<void>=>{
		if(!actions){
		return
		}
Expand DownExpand Up		@@ -45,7 +45,7 @@ export const setupActions = async ({ actions, send, path }: SetupActions): Promi

		// 4. run command
		if(!alreadyLoaded){
		awaitrunCommands({commands:commands\|\|[], send,path}).catch(onError)
		awaitrunCommands({commands:commands\|\|[], send,dir}).catch(onError)
		}
		}

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,12 +6,12 @@ import * as git from '../services/git'
		import { DISABLE_RUN_ON_SAVE } from '../environment'

		interface TutorialConfigParams {
		config: TT.TutorialConfig
		data: TT.Tutorial
		alreadyConfigured?: boolean
		onComplete?(): void
		}

		const tutorialConfig = async ({config, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage \| void> => {
		const tutorialConfig = async ({data, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage \| void> => {
		if (!alreadyConfigured) {
		// setup git, add remote
		const initError: E.ErrorMessage \| void = await git.initIfNotExists().catch(
Expand All		@@ -27,7 +27,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
		}

		// verify that internet is connected, remote exists and branch exists
		const remoteConnectError: E.ErrorMessage \| void = await git.checkRemoteConnects(config.repo).catch(
		const remoteConnectError: E.ErrorMessage \| void = await git.checkRemoteConnects(data.config.repo).catch(
		(error: Error): E.ErrorMessage => ({
		type: 'FailedToConnectToGitRepo',
		message: error.message,
Expand All		@@ -40,7 +40,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
		}

		// TODO if remote not already set
		const coderoadRemoteError: E.ErrorMessage \| void = await git.setupCodeRoadRemote(config.repo.uri).catch(
		const coderoadRemoteError: E.ErrorMessage \| void = await git.setupCodeRoadRemote(data.config.repo.uri).catch(
		(error: Error): E.ErrorMessage => ({
		type: 'GitRemoteAlreadyExists',
		message: error.message,
Expand All		@@ -52,7 +52,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
		}
		}

		await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER,config.testRunner)
		await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER,data)

		if (!DISABLE_RUN_ON_SAVE) {
		// verify if file test should run based on document saved
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -38,9 +38,11 @@ const loadWatchers = (watchers: string[]) => {
		constnow=+newDate()
		if(!lastFire\|\|lastFire-now>1000){
		vscode.commands.executeCommand(COMMANDS.RUN_TEST,{
		onSuccess:()=>{
		// cleanup watcher on success
		disposeWatcher(watcher)
		callbacks:{
		onSuccess:()=>{
		// cleanup watcher on success
		disposeWatcher(watcher)
		},
		},
		})
		}
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,10 +4,10 @@ import { exec } from '../../services/node'
		interfaceRunCommands{
		commands:string[]
		send:(action:T.Action)=>void
		path?:string
		dir?:string
		}

		construnCommands=async({ commands, send,path}:RunCommands)=>{
		construnCommands=async({ commands, send,dir}:RunCommands)=>{
		if(!commands.length){
		return
		}
Expand All		@@ -19,7 +19,7 @@ const runCommands = async ({ commands, send, path }: RunCommands) => {
		send({type:'COMMAND_START',payload:{process:{ ...process,status:'RUNNING'}}})
		letresult:{stdout:string;stderr:string}
		try{
		result=awaitexec({ command,path})
		result=awaitexec({ command,dir})
		console.log(result)
		}catch(error){
		console.log(`Test failed:${error.message}`)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -204,7 +204,7 @@ class Channel implements Channel {
		}
		}

		const error: E.ErrorMessage \| void = await tutorialConfig({config:data.config }).catch((error: Error) => ({
		const error: E.ErrorMessage \| void = await tutorialConfig({ data }).catch((error: Error) => ({
		type: 'UnknownError',
		message: `Location: tutorial config.\n\n${error.message}`,
		}))
Expand All		@@ -231,9 +231,8 @@ class Channel implements Channel {
		if (!tutorialContinue) {
		throw new Error('Invalid tutorial to continue')
		}
		const continueConfig: TT.TutorialConfig = tutorialContinue.config
		await tutorialConfig({
		config: continueConfig,
		data: tutorialContinue,
		alreadyConfigured: true,
		})
		// update the current stepId on startup
Expand DownExpand Up		@@ -307,7 +306,7 @@ class Channel implements Channel {
		await vscode.commands.executeCommand(COMMANDS.SET_CURRENT_POSITION, action.payload.position)
		await solutionActions({ actions: action.payload.actions, send: this.send })
		// run test following solution to update position
		vscode.commands.executeCommand(COMMANDS.RUN_TEST)
		vscode.commands.executeCommand(COMMANDS.RUN_TEST, { subtasks: true })
		return
		case 'EDITOR_SYNC_PROGRESS':
		// update progress when a level is deemed complete in the client
Expand All		@@ -318,7 +317,7 @@ class Channel implements Channel {
		await showOutput(channel)
		return
		case 'EDITOR_RUN_TEST':
		vscode.commands.executeCommand(COMMANDS.RUN_TEST)
		vscode.commands.executeCommand(COMMANDS.RUN_TEST, action?.payload)
		return
		default:
		logger(`No match for action type: ${actionType}`)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -50,13 +50,19 @@ export const createCommands = ({ extensionPath, workspaceState }: CreateCommandP
		// setup 1x1 horizontal layout
		webview.createOrShow()
		},
		[COMMANDS.CONFIG_TEST_RUNNER]: async (config: TT.TutorialTestRunnerConfig) => {
		if (config.actions) {
		[COMMANDS.CONFIG_TEST_RUNNER]: async (data: TT.Tutorial) => {
		const testRunnerConfig = data.config.testRunner
		const setup = testRunnerConfig.setup \|\| testRunnerConfig.actions // TODO: deprecate and remove config.actions
		if (setup) {
		// setup tutorial test runner commits
		// assumes git already exists
		await setupActions({ actions: config.actions, send: webview.send, path: config.path })
		await setupActions({
		actions: setup,
		send: webview.send,
		dir: testRunnerConfig.directory \|\| testRunnerConfig.path,
		}) // TODO: deprecate and remove config.path
		}
		testRunner = createTestRunner(config, {
		testRunner = createTestRunner(data, {
		onSuccess: (position: T.Position) => {
		logger('test pass position', position)
		// send test pass message back to client
Expand All		@@ -75,20 +81,27 @@ export const createCommands = ({ extensionPath, workspaceState }: CreateCommandP
		// send test run message back to client
		webview.send({ type: 'TEST_RUNNING', payload: { position } })
		},
		onLoadSubtasks: ({ summary }) => {
		webview.send({ type: 'LOAD_TEST_SUBTASKS', payload: { summary } })
		},
		})
		},
		[COMMANDS.SET_CURRENT_POSITION]: (position: T.Position) => {
		// set from last setup stepAction
		currentPosition = position
		},
		[COMMANDS.RUN_TEST]: (callback?: { onSuccess: () => void }) => {
		[COMMANDS.RUN_TEST]: ({
		subtasks,
		callbacks,
		}: { subtasks?: boolean; callbacks?: { onSuccess: () => void } } = {}) => {
		logger('run test current', currentPosition)
		// use stepId from client, or last set stepId
		// const position: T.Position = {
		// ...current,
		// stepId: current && current.position.stepId?.length ? current.position.stepId : currentPosition.stepId,
		// }
		testRunner(currentPosition, callback?.onSuccess)
		logger('currentPosition', currentPosition)
		testRunner({ position: currentPosition, onSuccess: callbacks?.onSuccess, subtasks })
		},
		}
		}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,11 +8,11 @@ const asyncExec = promisify(cpExec)

		interfaceExecParams{
		command:string
		path?:string
		dir?:string
		}

		exportconstexec=(params:ExecParams):Promise<{stdout:string;stderr:string}>\|never=>{
		constcwd=join(WORKSPACE_ROOT,params.path\|\|'')
		constcwd=join(WORKSPACE_ROOT,params.dir\|\|'')
		returnasyncExec(params.command,{ cwd})
		}

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,7 +2,7 @@ import * as T from 'typings'
		import * as TT from 'typings/tutorial'
		import { exec } from '../node'
		import logger from '../logger'
		import parser from './parser'
		import parser, { ParserOutput } from './parser'
		import { debounce, throttle } from './throttle'
		import onError from '../sentry/onError'
		import { clearOutput, addOutput } from './output'
Expand All		@@ -13,14 +13,22 @@ interface Callbacks {
		onFail(position: T.Position, failSummary: T.TestFail): void
		onRun(position: T.Position): void
		onError(position: T.Position): void
		onLoadSubtasks({ summary }: { summary: { [testName: string]: boolean } }): void
		}

		const failChannelName = 'CodeRoad (Tests)'
		const logChannelName = 'CodeRoad (Logs)'

		const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callbacks) => {
		return async (position: T.Position, onSuccess?: () => void): Promise<void> => {
		logger('createTestRunner', position)
		interface TestRunnerParams {
		position: T.Position
		subtasks?: boolean
		onSuccess?: () => void
		}

		const createTestRunner = (data: TT.Tutorial, callbacks: Callbacks) => {
		const testRunnerConfig = data.config.testRunner
		const testRunnerFilterArg = testRunnerConfig.args?.filter
		return async ({ position, onSuccess, subtasks }: TestRunnerParams): Promise<void> => {
		const startTime = throttle()
		// throttle time early
		if (!startTime) {
Expand All		@@ -30,11 +38,35 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
		logger('------------------- RUN TEST -------------------')

		// flag as running
		callbacks.onRun(position)
		if (!subtasks) {
		callbacks.onRun(position)
		}

		let result: { stdout: string \| undefined; stderr: string \| undefined }
		try {
		result = await exec({ command: config.command, path: config.path })
		let command = testRunnerConfig.args
		? `${testRunnerConfig.command} ${testRunnerConfig?.args.tap}`
		: testRunnerConfig.command // TODO: enforce TAP

		// filter tests if requested
		if (testRunnerFilterArg) {
		// get tutorial step from position
		// check the step actions for specific command
		// NOTE: cannot just pass in step actions as the test can be called by:
		// - onEditorSave, onWatcher, onSolution, onRunTest, onSubTask
		const levels = data.levels
		const level = levels.find((l) => l.id === position.levelId)
		const step = level?.steps.find((s) => s.id === position.stepId)
		const testFilter = step?.setup?.filter
		if (testFilter) {
		// append filter commands
		command = [command, testRunnerFilterArg, testFilter].join(' ')
		} else {
		throw new Error('Test Runner filter not configured')
		}
		}
		logger('COMMAND', command)
		result = await exec({ command, dir: testRunnerConfig.directory \|\| testRunnerConfig.path }) // TODO: remove config.path later
		} catch (err) {
		result = { stdout: err.stdout, stderr: err.stack }
		}
Expand All		@@ -49,7 +81,13 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba

		const { stdout, stderr } = result

		const tap = parser(stdout \|\| '')
		const tap: ParserOutput = parser(stdout \|\| '')

		if (subtasks) {
		callbacks.onLoadSubtasks({ summary: tap.summary })
		// exit early
		return
		}

		addOutput({ channel: logChannelName, text: tap.logs.join('\n'), show: false })

Expand All		@@ -60,6 +98,7 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
		const failSummary = {
		title: firstFail.message \|\| 'Test Failed',
		description: firstFail.details \|\| 'Unknown error',
		summary: tap.summary,
		}
		callbacks.onFail(position, failSummary)
		const output = formatFailOutput(tap)
Expand All		@@ -76,7 +115,9 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
		// PASS
		if (tap.ok) {
		clearOutput(failChannelName)

		callbacks.onSuccess(position)

		if (onSuccess) {
		onSuccess()
		}
Expand Down