@@ -125,3 +125,33 @@ CODEROAD_TUTORIAL_URL='path/to/tutorial_config_file.json' // will load directly
 ## [0.6.1]
 
 - Replace checkboxes with icons
+
+## [0.7.0]
+
+- Support loading subtasks (#340). Subtasks are a list of tests that need to pass before a task is complete. They can be loaded by:
+
+1. filtering down to a subset of tests by setting the `step.setup.filter` to a regex pattern that matches the tests you're targeting
+2. setting the `step.setup.subtasks` variable to true
+
+- Change for the test runner config. Changes are backwards compatible.
+
+1. `testRunner.path`=> `testRunner.directory`
+2. `testRunner.actions` => `testRunner.setup`
+3. Change command to capture `args` for "TAP" support, and test "filter"ing support. These changes will help lead to specific test suite presets in the future.
+
+```json
+{
+"testRunner": {
+"command": "mocha",
+"args": {
+"filter": "--grep",
+"tap": "--reporter=mocha-tap-reporter"
+},
+"directory": ".coderoad",
+"setup": {
+"commits": ["410bd4f"],
+"commands": ["npm install"]
+}
+}
+}
+```

@@ -45,7 +45,7 @@
 "@typescript-eslint/parser": "^2.33.0",
 "chokidar": "^3.4.0",
 "dotenv": "^8.2.0",
-"eslint": "^7.0.0",
+"eslint": "^6.8.0",
 "eslint-config-prettier": "^6.11.0",
 "eslint-plugin-prettier": "^3.1.3",
 "git-url-parse": "^11.1.2",

@@ -10,10 +10,10 @@ import logger from '../services/logger'
 interface SetupActions {
 actions: TT.StepActions
 send: (action: T.Action) => void // send messages to client
-path?: string
+dir?: string
 }
 
-export const setupActions = async ({ actions, send, path }: SetupActions): Promise<void> => {
+export const setupActions = async ({ actions, send, dir }: SetupActions): Promise<void> => {
 if (!actions) {
 return
 }
@@ -45,7 +45,7 @@ export const setupActions = async ({ actions, send, path }: SetupActions): Promi
 
 // 4. run command
 if (!alreadyLoaded) {
-await runCommands({ commands: commands || [], send, path }).catch(onError)
+await runCommands({ commands: commands || [], send, dir }).catch(onError)
 }
 }
 

@@ -6,12 +6,12 @@ import * as git from '../services/git'
 import { DISABLE_RUN_ON_SAVE } from '../environment'
 
 interface TutorialConfigParams {
-config: TT.TutorialConfig
+data: TT.Tutorial
 alreadyConfigured?: boolean
 onComplete?(): void
 }
 
-const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage | void> => {
+const tutorialConfig = async ({ data, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage | void> => {
 if (!alreadyConfigured) {
 // setup git, add remote
 const initError: E.ErrorMessage | void = await git.initIfNotExists().catch(
@@ -27,7 +27,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
 }
 
 // verify that internet is connected, remote exists and branch exists
-const remoteConnectError: E.ErrorMessage | void = await git.checkRemoteConnects(config.repo).catch(
+const remoteConnectError: E.ErrorMessage | void = await git.checkRemoteConnects(data.config.repo).catch(
 (error: Error): E.ErrorMessage => ({
 type: 'FailedToConnectToGitRepo',
 message: error.message,
@@ -40,7 +40,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
 }
 
 // TODO if remote not already set
-const coderoadRemoteError: E.ErrorMessage | void = await git.setupCodeRoadRemote(config.repo.uri).catch(
+const coderoadRemoteError: E.ErrorMessage | void = await git.setupCodeRoadRemote(data.config.repo.uri).catch(
 (error: Error): E.ErrorMessage => ({
 type: 'GitRemoteAlreadyExists',
 message: error.message,
@@ -52,7 +52,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
 }
 }
 
-await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER, config.testRunner)
+await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER, data)
 
 if (!DISABLE_RUN_ON_SAVE) {
 // verify if file test should run based on document saved

@@ -38,9 +38,11 @@ const loadWatchers = (watchers: string[]) => {
 const now = +new Date()
 if (!lastFire || lastFire - now > 1000) {
 vscode.commands.executeCommand(COMMANDS.RUN_TEST, {
-onSuccess: () => {
-// cleanup watcher on success
-disposeWatcher(watcher)
+callbacks: {
+onSuccess: () => {
+// cleanup watcher on success
+disposeWatcher(watcher)
+},
 },
 })
 }

@@ -4,10 +4,10 @@ import { exec } from '../../services/node'
 interface RunCommands {
 commands: string[]
 send: (action: T.Action) => void
-path?: string
+dir?: string
 }
 
-const runCommands = async ({ commands, send, path }: RunCommands) => {
+const runCommands = async ({ commands, send, dir }: RunCommands) => {
 if (!commands.length) {
 return
 }
@@ -19,7 +19,7 @@ const runCommands = async ({ commands, send, path }: RunCommands) => {
 send({ type: 'COMMAND_START', payload: { process: { ...process, status: 'RUNNING' } } })
 let result: { stdout: string; stderr: string }
 try {
-result = await exec({ command, path })
+result = await exec({ command, dir })
 console.log(result)
 } catch (error) {
 console.log(`Test failed: ${error.message}`)

@@ -204,7 +204,7 @@ class Channel implements Channel {
 }
 }
 
-const error: E.ErrorMessage | void = await tutorialConfig({ config: data.config }).catch((error: Error) => ({
+const error: E.ErrorMessage | void = await tutorialConfig({ data }).catch((error: Error) => ({
 type: 'UnknownError',
 message: `Location: tutorial config.\n\n${error.message}`,
 }))
@@ -231,9 +231,8 @@ class Channel implements Channel {
 if (!tutorialContinue) {
 throw new Error('Invalid tutorial to continue')
 }
-const continueConfig: TT.TutorialConfig = tutorialContinue.config
 await tutorialConfig({
-config: continueConfig,
+data: tutorialContinue,
 alreadyConfigured: true,
 })
 // update the current stepId on startup
@@ -307,7 +306,7 @@ class Channel implements Channel {
 await vscode.commands.executeCommand(COMMANDS.SET_CURRENT_POSITION, action.payload.position)
 await solutionActions({ actions: action.payload.actions, send: this.send })
 // run test following solution to update position
-vscode.commands.executeCommand(COMMANDS.RUN_TEST)
+vscode.commands.executeCommand(COMMANDS.RUN_TEST, { subtasks: true })
 return
 case 'EDITOR_SYNC_PROGRESS':
 // update progress when a level is deemed complete in the client
@@ -318,7 +317,7 @@ class Channel implements Channel {
 await showOutput(channel)
 return
 case 'EDITOR_RUN_TEST':
-vscode.commands.executeCommand(COMMANDS.RUN_TEST)
+vscode.commands.executeCommand(COMMANDS.RUN_TEST, action?.payload)
 return
 default:
 logger(`No match for action type: ${actionType}`)

@@ -50,13 +50,19 @@ export const createCommands = ({ extensionPath, workspaceState }: CreateCommandP
 // setup 1x1 horizontal layout
 webview.createOrShow()
 },
-[COMMANDS.CONFIG_TEST_RUNNER]: async (config: TT.TutorialTestRunnerConfig) => {
-if (config.actions) {
+[COMMANDS.CONFIG_TEST_RUNNER]: async (data: TT.Tutorial) => {
+const testRunnerConfig = data.config.testRunner
+const setup = testRunnerConfig.setup || testRunnerConfig.actions // TODO: deprecate and remove config.actions
+if (setup) {
 // setup tutorial test runner commits
 // assumes git already exists
-await setupActions({ actions: config.actions, send: webview.send, path: config.path })
+await setupActions({
+actions: setup,
+send: webview.send,
+dir: testRunnerConfig.directory || testRunnerConfig.path,
+}) // TODO: deprecate and remove config.path
 }
-testRunner = createTestRunner(config, {
+testRunner = createTestRunner(data, {
 onSuccess: (position: T.Position) => {
 logger('test pass position', position)
 // send test pass message back to client
@@ -75,20 +81,27 @@ export const createCommands = ({ extensionPath, workspaceState }: CreateCommandP
 // send test run message back to client
 webview.send({ type: 'TEST_RUNNING', payload: { position } })
 },
+onLoadSubtasks: ({ summary }) => {
+webview.send({ type: 'LOAD_TEST_SUBTASKS', payload: { summary } })
+},
 })
 },
 [COMMANDS.SET_CURRENT_POSITION]: (position: T.Position) => {
 // set from last setup stepAction
 currentPosition = position
 },
-[COMMANDS.RUN_TEST]: (callback?: { onSuccess: () => void }) => {
+[COMMANDS.RUN_TEST]: ({
+subtasks,
+callbacks,
+}: { subtasks?: boolean; callbacks?: { onSuccess: () => void } } = {}) => {
 logger('run test current', currentPosition)
 // use stepId from client, or last set stepId
 // const position: T.Position = {
 // ...current,
 // stepId: current && current.position.stepId?.length ? current.position.stepId : currentPosition.stepId,
 // }
-testRunner(currentPosition, callback?.onSuccess)
+logger('currentPosition', currentPosition)
+testRunner({ position: currentPosition, onSuccess: callbacks?.onSuccess, subtasks })
 },
 }
 }

@@ -8,11 +8,11 @@ const asyncExec = promisify(cpExec)
 
 interface ExecParams {
 command: string
-path?: string
+dir?: string
 }
 
 export const exec = (params: ExecParams): Promise<{ stdout: string; stderr: string }> | never => {
-const cwd = join(WORKSPACE_ROOT, params.path || '')
+const cwd = join(WORKSPACE_ROOT, params.dir || '')
 return asyncExec(params.command, { cwd })
 }
 

@@ -2,7 +2,7 @@ import * as T from 'typings'
 import * as TT from 'typings/tutorial'
 import { exec } from '../node'
 import logger from '../logger'
-import parser from './parser'
+import parser, { ParserOutput } from './parser'
 import { debounce, throttle } from './throttle'
 import onError from '../sentry/onError'
 import { clearOutput, addOutput } from './output'
@@ -13,14 +13,22 @@ interface Callbacks {
 onFail(position: T.Position, failSummary: T.TestFail): void
 onRun(position: T.Position): void
 onError(position: T.Position): void
+onLoadSubtasks({ summary }: { summary: { [testName: string]: boolean } }): void
 }
 
 const failChannelName = 'CodeRoad (Tests)'
 const logChannelName = 'CodeRoad (Logs)'
 
-const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callbacks) => {
-return async (position: T.Position, onSuccess?: () => void): Promise<void> => {
-logger('createTestRunner', position)
+interface TestRunnerParams {
+position: T.Position
+subtasks?: boolean
+onSuccess?: () => void
+}
+
+const createTestRunner = (data: TT.Tutorial, callbacks: Callbacks) => {
+const testRunnerConfig = data.config.testRunner
+const testRunnerFilterArg = testRunnerConfig.args?.filter
+return async ({ position, onSuccess, subtasks }: TestRunnerParams): Promise<void> => {
 const startTime = throttle()
 // throttle time early
 if (!startTime) {
@@ -30,11 +38,35 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
 logger('------------------- RUN TEST -------------------')
 
 // flag as running
-callbacks.onRun(position)
+if (!subtasks) {
+callbacks.onRun(position)
+}
 
 let result: { stdout: string | undefined; stderr: string | undefined }
 try {
-result = await exec({ command: config.command, path: config.path })
+let command = testRunnerConfig.args
+? `${testRunnerConfig.command} ${testRunnerConfig?.args.tap}`
+: testRunnerConfig.command // TODO: enforce TAP
+
+// filter tests if requested
+if (testRunnerFilterArg) {
+// get tutorial step from position
+// check the step actions for specific command
+// NOTE: cannot just pass in step actions as the test can be called by:
+// - onEditorSave, onWatcher, onSolution, onRunTest, onSubTask
+const levels = data.levels
+const level = levels.find((l) => l.id === position.levelId)
+const step = level?.steps.find((s) => s.id === position.stepId)
+const testFilter = step?.setup?.filter
+if (testFilter) {
+// append filter commands
+command = [command, testRunnerFilterArg, testFilter].join(' ')
+} else {
+throw new Error('Test Runner filter not configured')
+}
+}
+logger('COMMAND', command)
+result = await exec({ command, dir: testRunnerConfig.directory || testRunnerConfig.path }) // TODO: remove config.path later
 } catch (err) {
 result = { stdout: err.stdout, stderr: err.stack }
 }
@@ -49,7 +81,13 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
 
 const { stdout, stderr } = result
 
-const tap = parser(stdout || '')
+const tap: ParserOutput = parser(stdout || '')
+
+if (subtasks) {
+callbacks.onLoadSubtasks({ summary: tap.summary })
+// exit early
+return
+}
 
 addOutput({ channel: logChannelName, text: tap.logs.join('\n'), show: false })
 
@@ -60,6 +98,7 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
 const failSummary = {
 title: firstFail.message || 'Test Failed',
 description: firstFail.details || 'Unknown error',
+summary: tap.summary,
 }
 callbacks.onFail(position, failSummary)
 const output = formatFailOutput(tap)
@@ -76,7 +115,9 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
 // PASS
 if (tap.ok) {
 clearOutput(failChannelName)
+
 callbacks.onSuccess(position)
+
 if (onSuccess) {
 onSuccess()
 }

@@ -6,7 +6,13 @@ describe('parser', () => {
 1..1
 ok 1 - Should pass
 `
-expect(parser(example)).toEqual({ ok: true, passed: [{ message: 'Should pass' }], failed: [], logs: [] })
+expect(parser(example)).toEqual({
+ok: true,
+passed: [{ message: 'Should pass' }],
+failed: [],
+logs: [],
+summary: { 'Should pass': true },
+})
 })
 test('should detect multiple successes', () => {
 const example = `
@@ -20,6 +26,10 @@ ok 2 - Should also pass
 passed: [{ message: 'Should pass' }, { message: 'Should also pass' }],
 failed: [],
 logs: [],
+summary: {
+'Should pass': true,
+'Should also pass': true,
+},
 })
 })
 test('should detect failure if no tests passed', () => {
@@ -170,6 +180,10 @@ at processImmediate (internal/timers.js:439:21)`,
 },
 ],
 logs: ['log 1', 'log 2'],
+summary: {
+'package.json should have "express" installed': true,
+'server should log "Hello World"': false,
+},
 })
 })
 })