perf: run tests in parallel

fixes #12

cli/commands/generate/__test__/challenge.test.ts

@@ -22,7 +22,7 @@ await tap.test('programming-challenges generate challenge', async (t) => {
   t.beforeEach(() => {
     fsMock(
       {
-        [process.cwd()]: fsMock.load(process.cwd(), { recursive: true })
+        [process.cwd()]: fsMock.load(process.cwd(), { recursive: true, lazy: true })
       },
       { createCwd: false }
     )

cli/commands/run/__test__/test.test.ts

@@ -5,7 +5,7 @@ import sinon from 'sinon'
 import chalk from 'chalk'
 
 import { cli } from '../../../cli.js'
-import { Test } from '../../../services/Test.js'
+import { SolutionTestsResult } from '../../../services/SolutionTestsResult.js'
 
 const input = ['run', 'test']
 const challenge = 'hello-world'
@@ -46,7 +46,7 @@ await tap.test('programming-challenges run test', async (t) => {
       ),
       true
     )
-    t.equal(consoleLogSpy.calledWith(Test.SUCCESS_MESSAGE), true)
+    t.equal(consoleLogSpy.calledWith(SolutionTestsResult.SUCCESS_MESSAGE), true)
   })
 
   await t.test("fails with solution that doesn't exist", async (t) => {

cli/commands/run/solution.ts

@@ -8,6 +8,7 @@ import chalk from 'chalk'
 import { isExistingPath } from '../../utils/isExistingPath.js'
 import { template } from '../../services/Template.js'
 import { Solution } from '../../services/Solution.js'
+import { TemporaryFolder } from '../../services/TemporaryFolder.js'
 
 export class RunSolutionCommand extends Command {
   static paths = [['run', 'solution']]
@@ -47,6 +48,7 @@ export class RunSolutionCommand extends Command {
   async execute(): Promise<number> {
     console.log()
     try {
+      await TemporaryFolder.cleanAll()
       await template.verifySupportedProgrammingLanguage(
         this.programmingLanguage
       )
@@ -61,11 +63,13 @@ export class RunSolutionCommand extends Command {
       }
       const input = await fs.promises.readFile(inputPath, { encoding: 'utf-8' })
       await solution.run(input, this.output)
+      await TemporaryFolder.cleanAll()
       return 0
     } catch (error) {
       if (error instanceof Error) {
         console.error(`${chalk.bold.red('Error:')} ${error.message}`)
       }
+      await TemporaryFolder.cleanAll()
       return 1
     }
   }

cli/commands/run/test.ts

@@ -6,6 +6,8 @@ import { Solution } from '../../services/Solution.js'
 import { GitAffected } from '../../services/GitAffected.js'
 import { template } from '../../services/Template.js'
 import { Test } from '../../services/Test.js'
+import { SolutionTestsResult } from '../../services/SolutionTestsResult.js'
+import { TemporaryFolder } from '../../services/TemporaryFolder.js'
 
 export class RunTestCommand extends Command {
   static paths = [['run', 'test']]
@@ -38,10 +40,6 @@ export class RunTestCommand extends Command {
     description: 'Run the tests for all the solutions.'
   })
 
-  public isContinuousIntegration = Option.Boolean('--ci', false, {
-    description: 'Run the tests for the Continuous Integration (CI).'
-  })
-
   public base = Option.String('--base', {
     description: 'Base of the current branch (usually master)'
   })
@@ -49,17 +47,20 @@ export class RunTestCommand extends Command {
   async execute(): Promise<number> {
     console.log()
     try {
+      await TemporaryFolder.cleanAll()
       if (this.programmingLanguage != null) {
         await template.verifySupportedProgrammingLanguage(
           this.programmingLanguage
         )
       }
       if (this.all) {
-        return await Test.runAllTests(this.programmingLanguage)
+        const solutions = await Solution.getManyByProgrammingLanguages(
+          this.programmingLanguage != null ? [this.programmingLanguage] : undefined
+        )
+        return await Test.runManyWithSolutions(solutions)
       }
       if (this.affected) {
         const gitAffected = new GitAffected({
-          isContinuousIntegration: this.isContinuousIntegration,
           base: this.base
         })
         const solutions = await gitAffected.getAffectedSolutionsFromGit()
@@ -79,13 +80,22 @@ export class RunTestCommand extends Command {
         challengeName: this.challenge,
         programmingLanguageName: this.programmingLanguage
       })
-      await solution.test()
+      const result = await solution.test()
-      console.log(Test.SUCCESS_MESSAGE)
+      result.print({
+        shouldPrintBenchmark: true,
+        shouldPrintTableResult: true
+      })
+      await TemporaryFolder.cleanAll()
+      if (result.isSuccess)  {
+        console.log(SolutionTestsResult.SUCCESS_MESSAGE)
         return 0
+      }
+      return 1
     } catch (error) {
       if (error instanceof Error) {
         console.error(`${chalk.bold.red('Error:')} ${error.message}`)
       }
+      await TemporaryFolder.cleanAll()
       return 1
     }
   }

cli/services/Docker.ts

@@ -1,67 +1,78 @@
-import { performance } from 'node:perf_hooks'
-
 import { execaCommand } from 'execa'
-import ora from 'ora'
 import ms from 'ms'
 
+import { parseCommandOutput } from '../utils/parseCommandOutput.js'
+
 export interface DockerRunResult {
   stdout: string
-  elapsedTimeMilliseconds: number
 }
 
 export class Docker {
-  static CONTAINER_TAG = 'programming-challenges'
+  public static readonly CONTAINER_BASE_TAG = 'programming-challenges'
-  static SIGSEGV_EXIT_CODE = 139
+  public static readonly SIGSEGV_EXIT_CODE = 139
-  static MAXIMUM_TIMEOUT = '1 minute'
+  public static readonly MAXIMUM_TIMEOUT = '1 minute'
-  static MAXIMUM_TIMEOUT_MILLISECONDS = ms(Docker.MAXIMUM_TIMEOUT)
+  public static readonly MAXIMUM_TIMEOUT_MILLISECONDS = ms(
+    Docker.MAXIMUM_TIMEOUT
+  )
 
-  public async build(): Promise<void> {
+  public getContainerTag(id: string): string {
-    const loader = ora('Building the Docker image').start()
+    return `${Docker.CONTAINER_BASE_TAG}:${id}`
+  }
+
+  public async getImages(): Promise<string[]> {
     try {
-      await execaCommand(`docker build --tag=${Docker.CONTAINER_TAG} ./`)
+      const { stdout } = await execaCommand(
-      loader.stop()
+        `docker images -q --filter=reference="${Docker.CONTAINER_BASE_TAG}:*"`,
-    } catch (error) {
+        { shell: true }
-      loader.fail()
+      )
-      throw error
+      return parseCommandOutput(stdout)
+    } catch {
+      return []
     }
   }
 
-  public async run(input: string): Promise<DockerRunResult> {
+  public async removeImages(): Promise<void> {
+    try {
+      const images = await this.getImages()
+      if (images.length === 0) {
+        return
+      }
+      await execaCommand(`docker rmi -f ${images.join(' ')}`, {
+        shell: true
+      })
+    } catch {}
+  }
+
+  public async build(id: string): Promise<void> {
+    try {
+      await execaCommand(`docker build --tag=${this.getContainerTag(id)} ./`)
+    } catch (error: any) {
+      throw new Error(`Docker build failed.\n${error.message as string}`)
+    }
+  }
+
+  public async run(input: string, id: string): Promise<DockerRunResult> {
     const subprocess = execaCommand(
-      `docker run --interactive --rm ${Docker.CONTAINER_TAG}`,
+      `docker run --interactive --rm ${this.getContainerTag(id)}`,
       {
         input
       }
     )
-    let isValid = true
-    const timeout = setTimeout(() => {
-      subprocess.kill()
-      isValid = false
-    }, Docker.MAXIMUM_TIMEOUT_MILLISECONDS)
     try {
-      const start = performance.now()
       const { stdout, stderr } = await subprocess
-      const end = performance.now()
       if (stderr.length !== 0) {
         throw new Error(stderr)
       }
-      clearTimeout(timeout)
       return {
-        stdout,
+        stdout
-        elapsedTimeMilliseconds: end - start
       }
     } catch (error: any) {
-      if (!isValid) {
-        throw new Error(
-          `Timeout: time limit exceeded (${Docker.MAXIMUM_TIMEOUT}), try to optimize your solution.`
-        )
-      }
       if (error.exitCode === Docker.SIGSEGV_EXIT_CODE) {
         throw new Error(
-          "Docker run failed: SIGSEGV indicates a segmentation fault (attempts to access a memory location that it's not allowed to access)."
+          "Docker run failed.\nSIGSEGV indicates a segmentation fault (attempts to access a memory location that it's not allowed to access)."
         )
       }
-      throw new Error(`Docker run failed: ${error.message as string}`)
+      throw new Error(`Docker run failed.\n${error.message as string}`)
     }
   }
 }

cli/services/GitAffected.ts

@@ -2,6 +2,7 @@ import { execaCommand } from 'execa'
 
 import { Challenge } from './Challenge.js'
 import { Solution } from './Solution.js'
+import { parseCommandOutput } from '../utils/parseCommandOutput.js'
 
 const solutionsRegex =
   /challenges\/[\S\s]*\/solutions\/(c|cpp|cs|dart|java|javascript|python|rust|typescript)\/[\S\s]*\/(.*).(c|cpp|cs|dart|java|js|py|rs|ts)/
@@ -13,26 +14,16 @@ const inputOutputRegex =
   /challenges\/[\S\s]*\/test\/(.*)\/(input.txt|output.txt)/
 
 export interface GitAffectedOptions {
-  isContinuousIntegration: boolean
   base?: string
 }
 
 export class GitAffected implements GitAffectedOptions {
-  public isContinuousIntegration: boolean
   public base?: string
 
-  constructor(options: GitAffectedOptions) {
+  constructor(options: GitAffectedOptions = {}) {
-    this.isContinuousIntegration = options.isContinuousIntegration
     this.base = options.base
   }
 
-  public parseGitOutput(output: string): string[] {
-    return output
-      .split('\n')
-      .map((line) => line.trim())
-      .filter((line) => line.length > 0)
-  }
-
   public async getFilesUsingBaseAndHead(
     base: string,
     head: string
@@ -41,7 +32,7 @@ export class GitAffected implements GitAffectedOptions {
       const { stdout } = await execaCommand(
         `git diff --name-only --relative ${base} ${head}`
       )
-      return this.parseGitOutput(stdout)
+      return parseCommandOutput(stdout)
     } catch {
       return []
     }
@@ -52,8 +43,7 @@ export class GitAffected implements GitAffectedOptions {
   }
 
   public async getLatestPushedCommit(): Promise<string> {
-    const latestCommit =
+    const latestCommit = this.base != null ? '~1' : ''
-      this.isContinuousIntegration || this.base != null ? '~1' : ''
     const { stdout } = await execaCommand(
       `git rev-parse origin/master${latestCommit}`
     )

cli/services/Solution.ts

@@ -1,20 +1,19 @@
 import { fileURLToPath } from 'node:url'
 import path from 'node:path'
 import fs from 'node:fs'
+import { performance } from 'node:perf_hooks'
 
 import chalk from 'chalk'
 import ora from 'ora'
 
-import {
-  createTemporaryEmptyFolder,
-  TEMPORARY_PATH
-} from '../utils/createTemporaryEmptyFolder.js'
 import { isExistingPath } from '../utils/isExistingPath.js'
 import { Challenge } from './Challenge.js'
 import { copyDirectory } from '../utils/copyDirectory.js'
 import { template } from './Template.js'
 import { docker } from './Docker.js'
 import { Test } from './Test.js'
+import { SolutionTestsResult } from './SolutionTestsResult.js'
+import { TemporaryFolder } from './TemporaryFolder.js'
 
 export interface GetSolutionOptions {
   programmingLanguageName: string
@@ -37,6 +36,7 @@ export class Solution implements SolutionOptions {
   public challenge: Challenge
   public name: string
   public path: string
+  public temporaryFolder: TemporaryFolder
 
   constructor(options: SolutionOptions) {
     const { programmingLanguageName, challenge, name } = options
@@ -49,39 +49,50 @@ export class Solution implements SolutionOptions {
       programmingLanguageName,
       name
     )
+    this.temporaryFolder = new TemporaryFolder()
   }
 
-  private async prepareTemporaryFolder(): Promise<void> {
+  private async setup(): Promise<void> {
-    await createTemporaryEmptyFolder()
+    await this.temporaryFolder.create()
-    await copyDirectory(this.path, TEMPORARY_PATH)
+    await copyDirectory(this.path, this.temporaryFolder.path)
     await template.docker({
       programmingLanguage: this.programmingLanguageName,
-      destination: TEMPORARY_PATH
+      destination: this.temporaryFolder.path
     })
-    process.chdir(TEMPORARY_PATH)
+    process.chdir(this.temporaryFolder.path)
+    try {
+      await docker.build(this.temporaryFolder.id)
+    } catch (error: any) {
+      throw new Error(
+        `solution: ${this.path}\n${error.message as string}\n`
+      )
+    }
   }
 
-  public async test(): Promise<void> {
+  public async test(): Promise<SolutionTestsResult> {
-    await this.prepareTemporaryFolder()
+    await this.setup()
-    await docker.build()
+    return await Test.runAll(this)
-    await Test.runAll(this)
   }
 
   public async run(input: string, output: boolean = false): Promise<void> {
-    await this.prepareTemporaryFolder()
+    await this.setup()
-    await docker.build()
     const loader = ora('Running...').start()
     try {
-      const { stdout, elapsedTimeMilliseconds } = await docker.run(input)
+      const start = performance.now()
+      const { stdout } = await docker.run(input, this.temporaryFolder.id)
+      const end = performance.now()
+      const elapsedTimeMilliseconds = end - start
       loader.succeed(chalk.bold.green('Success!'))
-      Test.printBenchmark(elapsedTimeMilliseconds)
+      SolutionTestsResult.printBenchmark(elapsedTimeMilliseconds)
       if (output) {
         console.log(`${chalk.bold('Output:')}`)
         console.log(stdout)
       }
-    } catch (error) {
+    } catch (error: any) {
       loader.fail()
-      throw error
+      throw new Error(
+        `solution: ${this.path}\n${error.message as string}\n`
+      )
     }
   }
 

cli/services/SolutionTestsResult.ts

@@ -0,0 +1,95 @@
+import logSymbols from 'log-symbols'
+import chalk from 'chalk'
+import { table } from 'table'
+
+import type { Solution } from './Solution.js'
+import type { Test } from './Test.js'
+
+export interface SolutionTestsResultOptions {
+  tests: Test[]
+  solution: Solution
+  elapsedTimeMilliseconds: number
+}
+
+export interface SolutionTestsResultPrintOptions {
+  shouldPrintBenchmark?: boolean
+  shouldPrintTableResult?: boolean
+}
+
+export class SolutionTestsResult implements SolutionTestsResultOptions {
+  public tests: Test[] = []
+  public solution: Solution
+  public isSuccess: boolean
+  public elapsedTimeMilliseconds: number
+  public static readonly SUCCESS_MESSAGE = `${chalk.bold.green(
+    'Success:'
+  )} Tests passed! 🎉`
+
+  constructor(options: SolutionTestsResultOptions) {
+    this.tests = options.tests.sort((a, b) => {
+      return a.testNumber - b.testNumber
+    })
+    this.solution = options.solution
+    this.isSuccess = this.tests.every((test) => {
+      return test.isSuccess
+    })
+    this.elapsedTimeMilliseconds = options.elapsedTimeMilliseconds
+  }
+
+  public print(options: SolutionTestsResultPrintOptions = {}): void {
+    const { shouldPrintBenchmark = false, shouldPrintTableResult = false } = options
+    const name = `${this.solution.challenge.name}/${this.solution.programmingLanguageName}/${this.solution.name}`
+    console.log(`${chalk.bold('Name:')} ${name}\n`)
+    const tableResult = [
+      [
+        chalk.bold('N°'),
+        chalk.bold('Input'),
+        chalk.bold('Expected'),
+        chalk.bold('Received')
+      ]
+    ]
+    let totalFailedTests = 0
+    let totalCorrectTests = 0
+    for (const test of this.tests) {
+      const testLabel = `Test n°${test.testNumber}`
+      if (test.isSuccess) {
+        console.log(logSymbols.success, testLabel)
+        totalCorrectTests += 1
+      } else {
+        console.log(logSymbols.error, testLabel)
+        const expected = test.output.split('\n').join('\n')
+        const received = test.stdout.split('\n').join('\n')
+        tableResult.push([
+          test.testNumber.toString(),
+          `"${test.input}"`,
+          `"${expected}"`,
+          `"${received}"`
+        ])
+        totalFailedTests += 1
+      }
+    }
+    const isSuccess = totalCorrectTests === this.tests.length
+    console.log()
+    if (!isSuccess && shouldPrintTableResult) {
+      console.log(table(tableResult))
+    }
+    const testsResult = isSuccess
+      ? chalk.bold.green(`${totalCorrectTests} passed`)
+      : chalk.bold.red(`${totalFailedTests} failed`)
+    console.log(
+      `${chalk.bold('Tests:')} ${testsResult}, ${this.tests.length} total`
+    )
+    if (shouldPrintBenchmark) {
+      SolutionTestsResult.printBenchmark(this.elapsedTimeMilliseconds)
+    }
+    if (!isSuccess) {
+      throw new Error('Tests failed, try again!')
+    }
+    console.log('\n------------------------------\n')
+  }
+
+  public static printBenchmark(elapsedTimeMilliseconds: number): void {
+    const elapsedTime = elapsedTimeMilliseconds / 1000
+    console.log(`${chalk.bold('Benchmark:')} ${elapsedTime} seconds`)
+  }
+}

cli/services/Template.ts

@@ -109,7 +109,7 @@ class Template {
 
   public async getProgrammingLanguages(): Promise<string[]> {
     const languages = await fs.promises.readdir(TEMPLATE_SOLUTION_PATH)
-    return languages.filter((language) => language !== 'base')
+    return languages.filter((language) => {return language !== 'base'})
   }
 
   public async verifySupportedProgrammingLanguage(

cli/services/TemporaryFolder.ts

@@ -0,0 +1,31 @@
+import { fileURLToPath } from 'node:url'
+import fs from 'node:fs'
+import crypto from 'node:crypto'
+
+import { docker } from './Docker.js'
+
+export class TemporaryFolder {
+  public readonly id: string
+  public readonly path: string
+
+  public constructor() {
+    this.id = crypto.randomUUID()
+    this.path = fileURLToPath(new URL(`../../temp/${this.id}`, import.meta.url))
+  }
+
+  public async create(): Promise<void> {
+    await fs.promises.mkdir(this.path, { recursive: true })
+  }
+
+  public async delete(): Promise<void> {
+    await fs.promises.rm(this.path, { recursive: true, force: true })
+  }
+
+  public static async cleanAll(): Promise<void> {
+    try {
+      const temporaryPath = fileURLToPath(new URL('../../temp', import.meta.url))
+      await fs.promises.rm(temporaryPath, { recursive: true, force: true })
+      await docker.removeImages()
+    } catch {}
+  }
+}

cli/services/Test.ts

@@ -1,12 +1,13 @@
 import fs from 'node:fs'
 import path from 'node:path'
+import { performance } from 'node:perf_hooks'
 
-import ora from 'ora'
+import type { Solution } from './Solution.js'
-import chalk from 'chalk'
-import { table } from 'table'
-
-import { Solution } from './Solution.js'
 import { docker } from './Docker.js'
+import {
+  SolutionTestsResult
+} from './SolutionTestsResult.js'
+import { TemporaryFolder } from './TemporaryFolder.js'
 
 export interface InputOutput {
   input: string
@@ -14,118 +15,53 @@ export interface InputOutput {
 }
 
 export interface TestRunOptions {
-  index: number
+  testNumber: number
   path: string
+  solution: Solution
 }
 
 export interface TestOptions {
-  index: number
+  testNumber: number
   path: string
   isSuccess: boolean
   input: string
   output: string
   stdout: string
-  elapsedTimeMilliseconds: number
 }
 
 export class Test implements TestOptions {
-  public index: number
+  public testNumber: number
   public path: string
   public isSuccess: boolean
   public input: string
   public output: string
   public stdout: string
-  public elapsedTimeMilliseconds: number
-  static SUCCESS_MESSAGE = `${chalk.bold.green('Success:')} Tests passed! 🎉`
 
   constructor(options: TestOptions) {
-    this.index = options.index
+    this.testNumber = options.testNumber
     this.path = options.path
     this.isSuccess = options.isSuccess
     this.input = options.input
     this.output = options.output
     this.stdout = options.stdout
-    this.elapsedTimeMilliseconds = options.elapsedTimeMilliseconds
   }
 
-  static printResult(tests: Test[]): void {
+  static async runAll(solution: Solution): Promise<SolutionTestsResult> {
-    const tableResult = [
-      [
-        chalk.bold('N°'),
-        chalk.bold('Input'),
-        chalk.bold('Expected'),
-        chalk.bold('Received')
-      ]
-    ]
-    let totalFailedTests = 0
-    let totalCorrectTests = 0
-    let totalElapsedTimeMilliseconds = 0
-    for (const test of tests) {
-      if (!test.isSuccess) {
-        const expected = test.output.split('\n').join('"\n"')
-        const received = test.stdout.split('\n').join('"\n"')
-        tableResult.push([
-          test.index.toString(),
-          `"${test.input}"`,
-          `"${expected}"`,
-          `"${received}"`
-        ])
-        totalFailedTests += 1
-      } else {
-        totalCorrectTests += 1
-      }
-      totalElapsedTimeMilliseconds += test.elapsedTimeMilliseconds
-    }
-    const isSuccess = totalCorrectTests === tests.length
-    if (isSuccess) {
-      console.log()
-    } else {
-      console.log()
-      console.log(table(tableResult))
-    }
-    const testsResult = isSuccess
-      ? chalk.bold.green(`${totalCorrectTests} passed`)
-      : chalk.bold.red(`${totalFailedTests} failed`)
-    console.log(`${chalk.bold('Tests:')} ${testsResult}, ${tests.length} total`)
-    Test.printBenchmark(totalElapsedTimeMilliseconds)
-    if (!isSuccess) {
-      throw new Error('Tests failed, try again!')
-    }
-  }
-
-  static printBenchmark(elapsedTimeMilliseconds: number): void {
-    const elapsedTime = elapsedTimeMilliseconds / 1000
-    console.log(`${chalk.bold('Benchmark:')} ${elapsedTime} seconds`)
-  }
-
-  static async runAll(solution: Solution): Promise<void> {
-    const name = `${solution.challenge.name}/${solution.programmingLanguageName}/${solution.name}`
     const testsPath = path.join(solution.challenge.path, 'test')
     const testsFolders = await fs.promises.readdir(testsPath)
-    const testsNumbers = testsFolders
+    const testsNumbers = testsFolders.map((test) => {
-      .map((test) => Number(test))
+      return Number(test)
-      .sort((a, b) => a - b)
-    const tests: Test[] = []
-    console.log(`${chalk.bold('Name:')} ${name}\n`)
-    for (const testNumber of testsNumbers) {
-      const loader = ora(`Test n°${testNumber}`).start()
-      try {
-        const test = await Test.run({
-          path: path.join(testsPath, testNumber.toString()),
-          index: testNumber
     })
-        tests.push(test)
+    const testsPromises: Array<Promise<Test>> = []
-        if (test.isSuccess) {
+    const start = performance.now()
-          loader.succeed()
+    for (const testNumber of testsNumbers) {
-        } else {
+      const testPath = path.join(testsPath, testNumber.toString())
-          loader.fail()
+      testsPromises.push(Test.run({ testNumber, path: testPath, solution }))
     }
-      } catch (error) {
+    const tests = await Promise.all(testsPromises)
-        loader.fail()
+    const end = performance.now()
-        throw error
+    const elapsedTimeMilliseconds = end - start
-      }
+    return new SolutionTestsResult({ solution, tests, elapsedTimeMilliseconds })
-    }
-    Test.printResult(tests)
   }
 
   static async getInputOutput(testPath: string): Promise<InputOutput> {
@@ -138,35 +74,52 @@ export class Test implements TestOptions {
     return { input, output }
   }
 
-  static async runManyWithSolutions(solutions: Solution[]): Promise<number> {
+  static async runManyWithSolutions(
+    solutions: Solution[]
+  ): Promise<number> {
+    const solutionTestsResultsPromises: Array<Promise<SolutionTestsResult>> = []
+    let isSolutionSuccess = true
     for (const solution of solutions) {
-      await solution.test()
+      const solutionTestsResultPromise = solution.test()
-      console.log('\n------------------------------\n')
+      solutionTestsResultsPromises.push(solutionTestsResultPromise)
+      solutionTestsResultPromise
+        .then((solutionTestsResult) => {
+          solutionTestsResult.print()
+          if (!solutionTestsResult.isSuccess) {
+            isSolutionSuccess = false
           }
-    console.log(Test.SUCCESS_MESSAGE)
+        })
+        .catch(() => {})
+    }
+    await Promise.all(solutionTestsResultsPromises)
+    await TemporaryFolder.cleanAll()
+    if (isSolutionSuccess) {
+      console.log(SolutionTestsResult.SUCCESS_MESSAGE)
       return 0
     }
-
+    return 1
-  static async runAllTests(programmingLanguage?: string): Promise<number> {
-    const solutions = await Solution.getManyByProgrammingLanguages(
-      programmingLanguage != null ? [programmingLanguage] : undefined
-    )
-    await Test.runManyWithSolutions(solutions)
-    return 0
   }
 
   static async run(options: TestRunOptions): Promise<Test> {
     const { input, output } = await Test.getInputOutput(options.path)
-    const { stdout, elapsedTimeMilliseconds } = await docker.run(input)
+    try {
+      const { stdout } = await docker.run(
+        input,
+        options.solution.temporaryFolder.id
+      )
       const test = new Test({
         path: options.path,
-      index: options.index,
+        testNumber: options.testNumber,
         input,
         output,
         stdout,
-      isSuccess: stdout === output,
+        isSuccess: stdout === output
-      elapsedTimeMilliseconds
       })
       return test
+    } catch (error: any) {
+      throw new Error(
+        `solution: ${options.solution.path}\n${error.message as string}\n`
+      )
+    }
   }
 }

cli/services/__test__/GitAffected.test.ts

@@ -1,15 +1,29 @@
+import crypto from 'node:crypto'
+
 import tap from 'tap'
+import sinon from 'sinon'
 
 import { Challenge } from '../Challenge.js'
 import { GitAffected } from '../GitAffected.js'
 import { Solution } from '../Solution.js'
+import { parseCommandOutput } from '../../utils/parseCommandOutput.js'
 
-const gitAffected = new GitAffected({ isContinuousIntegration: false })
+const gitAffected = new GitAffected()
 
 await tap.test('services/GitAffected', async (t) => {
-  await t.test('parseGitOutput', async (t) => {
+  t.afterEach(() => {
+    sinon.restore()
+  })
+
+  t.beforeEach(() => {
+    sinon.stub(crypto, 'randomUUID').value(() => {
+      return 'uuid'
+    })
+  })
+
+  await t.test('parseCommandOutput', async (t) => {
     await t.test('returns the right output array', async (t) => {
-      t.same(gitAffected.parseGitOutput('1.txt\n 2.txt '), ['1.txt', '2.txt'])
+      t.same(parseCommandOutput('1.txt\n 2.txt '), ['1.txt', '2.txt'])
     })
   })
 

cli/utils/__test__/createTemporaryEmptyFolder.test.ts

@@ -1,38 +0,0 @@
-import fs from 'node:fs'
-
-import fsMock from 'mock-fs'
-import tap from 'tap'
-
-import {
-  TEMPORARY_PATH,
-  createTemporaryEmptyFolder
-} from '../createTemporaryEmptyFolder.js'
-import { isExistingPath } from '../isExistingPath.js'
-
-await tap.test('utils/createTemporaryEmptyFolder', async (t) => {
-  t.afterEach(() => {
-    fsMock.restore()
-  })
-
-  await t.test('should create the temporary folder', async (t) => {
-    fsMock({})
-    t.equal(await isExistingPath(TEMPORARY_PATH), false)
-    await createTemporaryEmptyFolder()
-    t.equal(await isExistingPath(TEMPORARY_PATH), true)
-  })
-
-  await t.test(
-    'should remove and create again the temporary folder',
-    async (t) => {
-      fsMock({
-        [TEMPORARY_PATH]: {
-          'file.txt': ''
-        }
-      })
-      t.equal(await isExistingPath(TEMPORARY_PATH), true)
-      t.equal((await fs.promises.readdir(TEMPORARY_PATH)).length, 1)
-      await createTemporaryEmptyFolder()
-      t.equal((await fs.promises.readdir(TEMPORARY_PATH)).length, 0)
-    }
-  )
-})

cli/utils/copyDirectory.ts

@@ -1,10 +1,10 @@
 import fs from 'node:fs'
 import path from 'node:path'
 
-export async function copyDirectory(
+export const copyDirectory = async (
   source: string,
   destination: string
-): Promise<void> {
+): Promise<void> => {
   const filesToCreate = await fs.promises.readdir(source)
   for (const file of filesToCreate) {
     const originalFilePath = path.join(source, file)

cli/utils/createTemporaryEmptyFolder.ts

@@ -1,14 +0,0 @@
-import { fileURLToPath } from 'node:url'
-import fs from 'node:fs'
-
-import { isExistingPath } from '../utils/isExistingPath.js'
-
-export const TEMPORARY_URL = new URL('../../temp', import.meta.url)
-export const TEMPORARY_PATH = fileURLToPath(TEMPORARY_URL)
-
-export const createTemporaryEmptyFolder = async (): Promise<void> => {
-  if (await isExistingPath(TEMPORARY_PATH)) {
-    await fs.promises.rm(TEMPORARY_URL, { recursive: true, force: true })
-  }
-  await fs.promises.mkdir(TEMPORARY_URL)
-}

cli/utils/parseCommandOutput.ts

@@ -0,0 +1,10 @@
+export const parseCommandOutput = (output: string): string[] => {
+  return output
+    .split('\n')
+    .map((line) => {
+      return line.trim()
+    })
+    .filter((line) => {
+      return line.length > 0
+    })
+}

package.json

@@ -30,29 +30,30 @@
     "clipanion": "3.0.1",
     "date-and-time": "2.4.1",
     "execa": "6.1.0",
+    "log-symbols": "5.1.0",
     "ora": "6.1.2",
     "replace-in-file": "6.3.5",
     "table": "6.8.0",
-    "typanion": "3.9.0",
+    "typanion": "3.12.0",
     "validate-npm-package-name": "4.0.0"
   },
   "devDependencies": {
     "@commitlint/cli": "17.1.2",
     "@commitlint/config-conventional": "17.1.0",
     "@swc/cli": "0.1.57",
-    "@swc/core": "1.2.245",
+    "@swc/core": "1.3.2",
-    "@types/sinon": "10.0.13",
-    "@types/tap": "15.0.7",
     "@types/date-and-time": "0.13.0",
     "@types/mock-fs": "4.13.1",
     "@types/ms": "0.7.31",
-    "@types/node": "18.7.14",
+    "@types/node": "18.7.18",
+    "@types/sinon": "10.0.13",
+    "@types/tap": "15.0.7",
     "@types/validate-npm-package-name": "4.0.0",
-    "@typescript-eslint/eslint-plugin": "5.36.0",
+    "@typescript-eslint/eslint-plugin": "5.38.0",
-    "@typescript-eslint/parser": "5.36.0",
+    "@typescript-eslint/parser": "5.38.0",
     "editorconfig-checker": "4.0.2",
-    "eslint": "8.23.0",
+    "eslint": "8.23.1",
-    "eslint-config-conventions": "3.0.0",
+    "eslint-config-conventions": "4.0.1",
     "eslint-plugin-import": "2.26.0",
     "eslint-plugin-promise": "6.0.1",
     "eslint-plugin-unicorn": "43.0.2",
@@ -63,6 +64,6 @@
     "rimraf": "3.0.2",
     "sinon": "14.0.0",
     "tap": "16.3.0",
-    "typescript": "4.8.2"
+    "typescript": "4.8.3"
   }
 }