Compare commits

..

6 Commits

Author SHA1 Message Date
bb39ae856d
docs: add notes about status of the project (abandoned for the moment)
All checks were successful
Chromatic / chromatic (push) Successful in 2m30s
CI / ci (push) Successful in 4m40s
CI / commitlint (push) Successful in 19s
2024-08-30 20:14:50 +02:00
170bdae725
build(deps): update latest 2024-08-30 20:06:53 +02:00
eba92ed64b
chore(storybook): next-intl Locale type 2024-08-25 02:00:02 +02:00
92787448bb
docs: database migrations 2024-08-25 01:50:29 +02:00
bf1729cf0d
perf(docker): optimize pnpm installation 2024-08-25 01:39:16 +02:00
f0b22f6a06
feat(api): work in progress GET /wikipedia/shortest-paths?fromPageId=id&toPageId=id 2024-08-23 23:21:42 +02:00
20 changed files with 1645 additions and 1226 deletions

View File

@ -1,7 +1,13 @@
# Wikipedia Game Solver
> \[!IMPORTANT\]
> This project is a work in progress, at an early stage of development.
> The project is abandoned and not maintained anymore.
>
> It was a proof of concept to deal with large amounts of data in databases, and have some fun dealing with Wikipedia data.
>
> The project was also a way to learn and experiment with a clean monorepo architecture, with [Turborepo](https://turbo.build/repo), and [TypeScript](https://www.typescriptlang.org/) as the main language.
>
> The project setup **can be reused as a template/boilerplate for future projects**.
## About
@ -11,17 +17,12 @@ The Wikipedia Game involves players competing to navigate from one [Wikipedia](h
Available online: <https://wikipedia-game-solver.theoludwig.fr>
> \[!NOTE\]
> The project is also a way to learn and experiment with a monorepo architecture, with [Turborepo](https://turbo.build/repo), and [TypeScript](https://www.typescriptlang.org/) as the main language.
>
> The project setup **can be used as a template/boilerplate for new projects**.
## Getting Started
### Prerequisites
- [Node.js](https://nodejs.org/) >= 22.0.0
- [pnpm](https://pnpm.io/) >= 9.5.0
- [pnpm](https://pnpm.io/) >= 9.9.0
- [Docker](https://www.docker.com/)
### Installation
@ -51,6 +52,9 @@ node --run dev
# Start the development Docker services (e.g: Database)
docker compose --file compose.dev.yaml up
# Database migrations
node --run database:migrate
# Lint
node --run lint:editorconfig
node --run lint:prettier
@ -73,6 +77,10 @@ node --run ace -- list
```sh
# Setup and run all the services for you
docker compose up --build
# To execute database migrations
docker compose exec wikipedia-game-solver-api sh
node --run database:migrate
```
#### Services started

View File

@ -33,7 +33,7 @@
- [x] Create Lucid models and migrations for Wikipedia Database Dump: `pages` and `internal_links` tables
- [x] Implement `GET /wikipedia/pages?title=Node.js` to search a page by title (not necessarily with the title sanitized, search with input by user to check if page exists)
- [x] Implement `GET /wikipedia/pages/[id]` to get a page and all its internal links with the pageId
- [ ] Implement `GET /wikipedia/internal-links/paths?fromPageId=id&toPageId=id` to get all the possible paths between 2 pages
- [ ] Implement `GET /wikipedia/shortest-paths?fromPageId=id&toPageId=id` to get all the possible paths between 2 pages (e.g: `Node.js` `26415635` => `Linux` `6097297`)
- [x] Setup tests with database + add coverage
- [x] Setup Health checks
- [x] Setup Rate limiting
@ -48,7 +48,7 @@
- [ ] Implement CLI (`cli`)
- [ ] Init Clipanion project
- [ ] Implement `wikipedia-game-solver internal-links --from="Node.js" --to="Linux"` command to get all the possible paths between 2 pages.
- [ ] Add docs to add locale/edit translations, create component, install a dependency in a package, create a new package, technology used, architecture, links where it's deployed, how to use/install for end users, how to update dependencies with `npx taze -l` etc.
- [ ] Add docs to add locale/edit translations, create component, install a dependency in a package, create a new package, technology used, architecture, links where it's deployed, how to use/install for end users, how to update dependencies with `npx taze -l major` etc.
- [ ] GitHub Mirror
- [ ] Delete `TODO.md` file and instead use issues for the remaining tasks

View File

@ -11,5 +11,6 @@ DATABASE_PASSWORD=password
DATABASE_NAME=wikipedia
DATABASE_HOST=127.0.0.1
DATABASE_PORT=5432
DATABASE_DEBUG=false
LIMITER_STORE=database

View File

@ -2,11 +2,13 @@ FROM node:22.4.1-slim AS node-pnpm
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
ENV TURBO_TELEMETRY_DISABLED=1
ENV DO_NOT_TRACK=1
WORKDIR /usr/src/app
FROM node-pnpm AS builder
RUN pnpm install --global turbo@2.0.14
COPY ./ ./
RUN pnpm install --global turbo@2.1.0
RUN turbo prune @repo/api --docker
FROM node-pnpm AS installer

View File

@ -0,0 +1,226 @@
import type { ExceptionMessage } from "#app/exceptions/handler.ts"
import Page, { type PageRaw } from "#app/models/page.ts"
import { throttle } from "#start/limiter.ts"
import type { HttpContext } from "@adonisjs/core/http"
import router from "@adonisjs/core/services/router"
import vine from "@vinejs/vine"
export const get_shortest_paths_validator = vine.compile(
vine.object({
fromPageId: vine.number().withoutDecimals().positive(),
toPageId: vine.number().withoutDecimals().positive(),
}),
)
interface get_shortest_paths_response {
/**
* Object to get page information by their id.
* - Key: Page id.
* - Value: Page information.
*/
pages: Record<number, PageRaw>
/**
* Paths between two pages using only internal links.
* Each path is an array of page ids.
*/
paths: number[]
}
const getPathsBetweenPages = async (
fromPage: Page,
toPage: Page,
): Promise<number[]> => {
if (fromPage.id === toPage.id) {
return [fromPage.id]
}
// A queue of paths from the start page
const forwardQueue: Array<[Page, number[]]> = [[fromPage, [fromPage.id]]]
// A queue of paths from the end page
const backwardQueue: Array<[Page, number[]]> = [[toPage, [toPage.id]]]
// Sets to track visited pages from both ends
const visitedFromStart = new Set<number>([fromPage.id])
const visitedFromEnd = new Set<number>([toPage.id])
// Maps to track paths
const forwardPaths = new Map<number, number[]>([[fromPage.id, [fromPage.id]]])
const backwardPaths = new Map<number, number[]>([[toPage.id, [toPage.id]]])
// Helper function to process a queue in one direction
const processQueue = async (
queue: Array<[Page, number[]]>,
visitedThisSide: Set<number>,
visitedOtherSide: Set<number>,
pathsThisSide: Map<number, number[]>,
pathsOtherSide: Map<number, number[]>,
): Promise<number[]> => {
const [currentPage, currentPath] = queue.shift() as [Page, number[]]
await currentPage.load("internalLinks")
for (const link of currentPage.internalLinks) {
if (!visitedThisSide.has(link.id)) {
const newPath = [...currentPath, link.id]
visitedThisSide.add(link.id)
pathsThisSide.set(link.id, newPath)
// If the other side has visited this page, we've found a meeting point
if (visitedOtherSide.has(link.id)) {
const pathFromOtherSide = pathsOtherSide.get(link.id) ?? []
return [...newPath.slice(0, -1), ...pathFromOtherSide.reverse()]
}
// Otherwise, continue the BFS
queue.push([link, newPath])
}
}
return []
}
while (forwardQueue.length > 0 && backwardQueue.length > 0) {
// Expand the BFS from the start side
let result = await processQueue(
forwardQueue,
visitedFromStart,
visitedFromEnd,
forwardPaths,
backwardPaths,
)
if (result.length > 0) {
return result
}
// Expand the BFS from the end side
result = await processQueue(
backwardQueue,
visitedFromEnd,
visitedFromStart,
backwardPaths,
forwardPaths,
)
if (result.length > 0) {
return result
}
}
return []
}
// const getPathsBetweenPages = async (
// fromPage: Page,
// toPage: Page,
// ): Promise<number[][]> => {
// const paths: number[][] = []
// const depthFirstSearch = async (
// currentPage: Page,
// currentPath: number[],
// ): Promise<void> => {
// currentPath.push(currentPage.id)
// if (currentPage.id === toPage.id) {
// paths.push([...currentPath])
// } else {
// for (const link of currentPage.internalLinks) {
// const isAlreadyVisited = currentPath.includes(link.id)
// if (!isAlreadyVisited) {
// await link.load("internalLinks")
// await depthFirstSearch(link, currentPath)
// }
// }
// }
// currentPath.pop()
// }
// await depthFirstSearch(fromPage, [])
// return paths
// }
export default class get_shortest_paths {
public async handle(context: HttpContext): Promise<
| {
__response: ExceptionMessage
__status: 404
}
| {
__response: ExceptionMessage
__status: 500
}
| {
__response: get_shortest_paths_response
__status: 200
}
> {
const payload = await context.request.validateUsing(
get_shortest_paths_validator,
)
const fromPage = await Page.findOrFail(payload.fromPageId)
await fromPage.load("internalLinks")
const toPage = await Page.findOrFail(payload.toPageId)
await toPage.load("internalLinks")
const isDepth0 = fromPage.id === toPage.id
if (isDepth0) {
return context.response.ok({
pages: {
[fromPage.id]: {
id: fromPage.id,
title: fromPage.title,
},
},
paths: [],
})
}
const isDepth1 = fromPage.internalLinks.some((internalLink) => {
return internalLink.id === toPage.id
})
if (isDepth1) {
return context.response.ok({
pages: {
[fromPage.id]: {
id: fromPage.id,
title: fromPage.title,
},
[toPage.id]: {
id: toPage.id,
title: toPage.title,
},
},
paths: [fromPage.id, toPage.id],
})
}
const paths = await getPathsBetweenPages(fromPage, toPage)
return context.response.ok({
pages: {
[fromPage.id]: {
id: fromPage.id,
title: fromPage.title,
},
[toPage.id]: {
id: toPage.id,
title: toPage.title,
},
},
paths,
})
// return context.response.internalServerError({
// message: "Shortest paths can't be determined.",
// })
}
}
router
.get("/wikipedia/shortest-paths", [get_shortest_paths])
.use(throttle)
.openapi({
description:
"Find the shortest paths between two Wikipedia pages, using only internal links.",
tags: ["wikipedia"],
})

View File

@ -7,7 +7,7 @@ const databaseConfig = defineConfig({
connection: app.inTest ? "sqlite" : "postgres",
connections: {
postgres: {
debug: app.inDev,
debug: env.get("DATABASE_DEBUG"),
client: "pg",
connection: {
host: env.get("DATABASE_HOST"),

View File

@ -15,7 +15,9 @@
"ace": "node --import=tsx ./bin/console.ts",
"tuyau": "node --run ace -- tuyau:generate && node --run ace -- tuyau:generate:openapi --destination=\".adonisjs/openapi.yaml\"",
"build": "node --run tuyau",
"database:migrate": "node --run ace -- migration:run",
"test": "c8 node --import=tsx ./bin/test.ts",
"test-shortest-paths": "node --import=tsx ./shortest-paths-tests.ts",
"lint:eslint": "eslint . --max-warnings 0 --report-unused-disable-directives",
"lint:typescript": "tsc --noEmit"
},

View File

@ -0,0 +1,170 @@
import { type PageWithInternalLinksRaw } from "#app/models/page.ts"
const DATA: { [key: number]: PageWithInternalLinksRaw } = {
0: {
id: 0,
title: "Page 0",
internalLinks: [
{
id: 1,
title: "Page 1",
},
{
id: 4,
title: "Page 4",
},
],
},
1: {
id: 1,
title: "Page 1",
internalLinks: [
{
id: 2,
title: "Page 2",
},
],
},
2: {
id: 2,
title: "Page 2",
internalLinks: [
{
id: 1,
title: "Page 1",
},
{
id: 3,
title: "Page 3",
},
{
id: 4,
title: "Page 4",
},
],
},
3: {
id: 3,
title: "Page 3",
internalLinks: [],
},
4: {
id: 4,
title: "Page 4",
internalLinks: [
{
id: 2,
title: "Page 2",
},
{
id: 3,
title: "Page 3",
},
],
},
}
// PILE (stack): LIFO: .pop()
// FILE (queue): FIFO: .shift()
// parcours en profondeur, ou DFS, pour Depth-First Search
// get all possible paths from 0 to 3
// [[0, 1, 2, 3], [0, 1, 2, 4, 3], [0, 4, 2, 3], [0, 4, 3]]
// console.log(DATA)
const getPathsBetweenPages = async (
fromPage: PageWithInternalLinksRaw,
toPage: PageWithInternalLinksRaw,
getPageById: (id: number) => Promise<PageWithInternalLinksRaw>,
): Promise<number[][]> => {
const paths: number[][] = []
const depthFirstSearch = async (
currentPage: PageWithInternalLinksRaw,
currentPath: number[],
): Promise<void> => {
currentPath.push(currentPage.id)
if (currentPage.id === toPage.id) {
paths.push([...currentPath])
} else {
for (const link of currentPage.internalLinks) {
const isAlreadyVisited = currentPath.includes(link.id)
if (!isAlreadyVisited) {
await depthFirstSearch(await getPageById(link.id), currentPath)
}
}
}
currentPath.pop()
}
await depthFirstSearch(fromPage, [])
return paths
}
const getShortestPathsBetweenPages = async (
fromPage: PageWithInternalLinksRaw,
toPage: PageWithInternalLinksRaw,
getPageById: (id: number) => Promise<PageWithInternalLinksRaw>,
): Promise<number[][]> => {
const shortestPaths: number[][] = []
const queue: Array<{ page: PageWithInternalLinksRaw; path: number[] }> = [
{ page: fromPage, path: [fromPage.id] },
]
let shortestLength: number | null = null
while (queue.length > 0) {
const { page, path } = queue.shift() as {
page: PageWithInternalLinksRaw
path: number[]
}
if (page.id === toPage.id) {
// If we reached the destination, check the path length
if (shortestLength === null || path.length <= shortestLength) {
if (shortestLength === null) {
shortestLength = path.length
}
if (path.length === shortestLength) {
shortestPaths.push(path)
}
}
// If we found a shorter path, discard previously found paths
else if (path.length < shortestLength) {
shortestPaths.length = 0
shortestPaths.push(path)
shortestLength = path.length
}
} else {
for (const link of page.internalLinks) {
if (!path.includes(link.id)) {
queue.push({
page: await getPageById(link.id),
path: [...path, link.id],
})
}
}
}
}
return shortestPaths
}
console.log(
await getPathsBetweenPages(
DATA[0] as PageWithInternalLinksRaw,
DATA[3] as PageWithInternalLinksRaw,
async (id) => {
return DATA[id] as PageWithInternalLinksRaw
},
),
)
console.log(
await getShortestPathsBetweenPages(
DATA[0] as PageWithInternalLinksRaw,
DATA[3] as PageWithInternalLinksRaw,
async (id) => {
return DATA[id] as PageWithInternalLinksRaw
},
),
)

View File

@ -28,6 +28,7 @@ export default await Env.create(new URL("..", import.meta.url), {
DATABASE_USER: Env.schema.string(),
DATABASE_PASSWORD: Env.schema.string(),
DATABASE_NAME: Env.schema.string(),
DATABASE_DEBUG: Env.schema.boolean(),
/**
* Variables for configuring the limiter package.

View File

@ -1,6 +1,7 @@
import "@repo/config-tailwind/styles.css"
import { defaultTranslationValues, Locale } from "@repo/i18n/config"
import { defaultTranslationValues } from "@repo/i18n/config"
import i18nMessagesEnglish from "@repo/i18n/translations/en-US.json"
import type { Locale } from "@repo/utils/constants"
import type { Preview } from "@storybook/react"
import { NextIntlClientProvider } from "next-intl"
import { ThemeProvider as NextThemeProvider } from "next-themes"

View File

@ -5,7 +5,6 @@ import { checkA11y, configureAxe, injectAxe } from "axe-playwright"
/*
* See https://storybook.js.org/docs/writing-tests/test-runner#test-hook-api
* to learn more about the test-runner hooks API.
*/
const config: TestRunnerConfig = {
async preVisit(page) {

View File

@ -16,6 +16,7 @@
"@repo/config-tailwind": "workspace:*",
"@repo/i18n": "workspace:*",
"@repo/ui": "workspace:*",
"@repo/utils": "workspace:*",
"@repo/wikipedia-game-solver": "workspace:*",
"next": "catalog:",
"next-intl": "catalog:",

View File

@ -1,3 +1,4 @@
TZ=UTC
HOSTNAME=0.0.0.0
PORT=5000
NEXT_TELEMETRY_DISABLED=1

View File

@ -2,11 +2,13 @@ FROM node:22.4.1-slim AS node-pnpm
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
ENV TURBO_TELEMETRY_DISABLED=1
ENV DO_NOT_TRACK=1
WORKDIR /usr/src/app
FROM node-pnpm AS builder
RUN pnpm install --global turbo@2.0.14
COPY ./ ./
RUN pnpm install --global turbo@2.1.0
RUN turbo prune @repo/website --docker
FROM node-pnpm AS installer

View File

@ -36,8 +36,6 @@ services:
POSTGRES_DB: ${DATABASE_NAME}
command: |
--max_wal_size=4GB
ports:
- "${DATABASE_PORT-5432}:${DATABASE_PORT-5432}"
volumes:
- "wikipedia-solver-postgres-data:/var/lib/postgresql/data"
- "./data:/data/"

View File

@ -1,4 +1,16 @@
{
"root": true,
"extends": ["@repo/eslint-config"]
"extends": ["@repo/eslint-config"],
"rules": {
"import-x/extensions": [
"error",
"ignorePackages",
{
"ts": "never",
"tsx": "never",
"js": "always",
"jsx": "never"
}
]
}
}

View File

@ -3,14 +3,15 @@
"version": "1.0.0-staging.3",
"private": true,
"type": "module",
"packageManager": "pnpm@9.5.0+sha512.140036830124618d624a2187b50d04289d5a087f326c9edfc0ccd733d76c4f52c3a313d4fc148794a2a9d81553016004e6742e8cf850670268a7387fc220c903",
"packageManager": "pnpm@9.9.0+sha512.60c18acd138bff695d339be6ad13f7e936eea6745660d4cc4a776d5247c540d0edee1a563695c183a66eb917ef88f2b4feb1fc25f32a7adcadc7aaf3438e99c1",
"engines": {
"node": ">=22.0.0",
"pnpm": ">=9.5.0"
"pnpm": ">=9.9.0"
},
"scripts": {
"dev": "turbo run dev --parallel",
"start": "turbo run start --parallel",
"database:migrate": "turbo run database:migrate",
"build": "turbo run build",
"test": "turbo run test",
"lint:editorconfig": "editorconfig-checker",
@ -29,7 +30,7 @@
"prettier-plugin-tailwindcss": "0.6.6",
"replace-in-files-cli": "3.0.0",
"semantic-release": "23.1.1",
"turbo": "2.0.14",
"turbo": "2.1.0",
"typescript": "catalog:"
},
"pnpm": {

2371
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -5,24 +5,24 @@ packages:
catalog:
# Utils
"deepmerge": "4.3.1"
"ky": "1.6.0"
"ky": "1.7.1"
# React.js/Next.js
"next": "14.2.5"
"next-intl": "3.17.3"
"next": "14.2.7"
"next-intl": "3.19.0"
"next-themes": "0.3.0"
"react": "18.3.1"
"react-dom": "18.3.1"
"react-icons": "5.3.0"
"@types/react": "18.3.3"
"@types/react": "18.3.5"
"@types/react-dom": "18.3.0"
"sharp": "0.33.4"
"sharp": "0.33.5"
# TypeScript
"typescript": "5.5.4"
"@total-typescript/ts-reset": "0.5.1"
"@types/node": "22.3.0"
"tsx": "4.17.0"
"@total-typescript/ts-reset": "0.6.0"
"@types/node": "22.5.1"
"tsx": "4.19.0"
# AdonisJS
"@adonisjs/auth": "9.2.3"
@ -31,7 +31,7 @@ catalog:
"@adonisjs/lucid": "21.2.0"
"@adonisjs/limiter": "2.3.2"
"pg": "8.12.0"
"better-sqlite3": "11.1.2"
"better-sqlite3": "11.2.1"
"@adonisjs/assembler": "7.7.0"
"@vinejs/vine": "2.1.0"
"luxon": "3.5.0"
@ -57,13 +57,13 @@ catalog:
"eslint-config-conventions": "14.4.0"
"eslint-plugin-promise": "7.1.0"
"eslint-plugin-unicorn": "55.0.0"
"eslint-config-next": "14.2.5"
"eslint-config-next": "14.2.7"
"eslint-plugin-storybook": "0.8.0"
"eslint-plugin-tailwindcss": "3.17.4"
"eslint-plugin-import-x": "3.1.0"
# Storybook
"@chromatic-com/storybook": "1.6.1"
"@chromatic-com/storybook": "1.8.0"
"@storybook/addon-a11y": "8.2.9"
"@storybook/addon-essentials": "8.2.9"
"@storybook/addon-interactions": "8.2.9"
@ -81,21 +81,21 @@ catalog:
"storybook-dark-mode": "4.0.2"
# Testing
"playwright": "1.46.0"
"@playwright/test": "1.46.0"
"axe-playwright": "2.0.1"
"playwright": "1.46.1"
"@playwright/test": "1.46.1"
"axe-playwright": "2.0.2"
"start-server-and-test": "2.0.5"
"@vitest/browser": "2.0.5"
"@vitest/coverage-istanbul": "2.0.5"
"c8": "10.1.2"
"@vitest/ui": "2.0.5"
"vitest": "2.0.5"
"@testing-library/react": "16.0.0"
"@testing-library/react": "16.0.1"
# CSS
"postcss": "8.4.41"
"tailwindcss": "3.4.10"
"@fontsource/montserrat": "5.0.18"
"@fontsource/montserrat": "5.0.19"
"clsx": "2.1.0"
"cva": "1.0.0-beta.1"
"tailwind-merge": "2.5.2"

View File

@ -31,6 +31,9 @@
"start": {
"dependsOn": ["build"],
"persistent": true
},
"database:migrate": {
"cache": false
}
}
}