feat: try deep internal links of wikipedia pages

TODO.md

@@ -2,14 +2,12 @@
 
 - [x] chore: initial commit (+ mirror on GitHub)
 - [x] Deploy first staging version (v1.0.0-staging.1)
-- [ ] Add docs to add locale/edit translations, create component, install a dependency in a package, create a new package, technology used, architecture, links where it's deployed, how to use/install for end users, how to update dependencies with `npx taze -l` etc.
+- [ ] Implement Wikipedia Game Solver (`website`) with inputs, button to submit, and list all pages to go from one to another, or none if it is not possible
-- [ ] Implement Wikipedia Game Solver (`website`) with inputs, button to submit, and list all articles to go from one to another, or none if it is not possible
+- [ ] Check, cache and store (in `.json` file) all Wikipedia Pages and its internal links, maybe use Wikipedia Dump (<https://en.wikipedia.org/wiki/Wikipedia:Database_download>)?
-- [ ] v1.0.0-staging.2
+- [ ] Implement toast notifications for errors, warnings, and success messages
 - [ ] Implement CLI (`cli`)
-- [ ] v1.0.0-staging.3
 - [ ] Implement REST API (`api`) with JSON responses ([AdonisJS](https://adonisjs.com/))
-- [ ] v1.0.0-staging.4
+- [ ] Add docs to add locale/edit translations, create component, install a dependency in a package, create a new package, technology used, architecture, links where it's deployed, how to use/install for end users, how to update dependencies with `npx taze -l` etc.
-- [ ] v1.0.0
 
 ## Links
 

apps/cli/package.json

@@ -11,7 +11,7 @@
   },
   "scripts": {
     "start": "node --import=tsx ./src/index.ts",
-    "dev": "node --import=tsx --watch --watch-preserve-output ./src/index.ts",
+    "dev-test": "node --import=tsx --watch --watch-preserve-output ./src/index.ts",
     "lint:eslint": "eslint src --max-warnings 0 --report-unused-disable-directives",
     "lint:typescript": "tsc --noEmit"
   },

apps/cli/src/index.ts

@@ -1,11 +1,26 @@
 #!/usr/bin/env -S node --import=tsx
 
-import { add } from "#abc/def/add.js"
+import { getWikipediaPageInternalLinks } from "@repo/wikipedia-game-solver/wikipedia-api"
 
-import { VERSION } from "@repo/constants"
+const localeWikipedia = "en"
-import { sum } from "@repo/wikipedia-game-solver/wikipedia-api"
 
-console.log("Hello, world!")
+const fromPageInput = "Linux"
-console.log(sum(1, 2))
+const toPageInput = "Node.js"
-console.log(add(2, 3))
+console.log({
-console.log(`v${VERSION}`)
+  fromPageInput,
+  toPageInput,
+})
+const [fromPageWikipediaLinks, toPageWikipediaLinks] = await Promise.all([
+  getWikipediaPageInternalLinks({
+    title: fromPageInput,
+    locale: localeWikipedia,
+  }),
+  getWikipediaPageInternalLinks({
+    title: toPageInput,
+    locale: localeWikipedia,
+  }),
+])
+console.log({
+  fromPageWikipediaLinks,
+  toPageWikipediaLinks,
+})

apps/cli/src/main.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env -S node --import=tsx
+
+import { add } from "#abc/def/add.js"
+
+import { VERSION } from "@repo/constants"
+import { sum } from "@repo/wikipedia-game-solver/wikipedia-api"
+
+console.log("Hello, world!")
+console.log(sum(1, 2))
+console.log(add(2, 3))
+console.log(`v${VERSION}`)

packages/config-tailwind/tailwind.config.js

@@ -23,6 +23,7 @@ const config = {
       yellow: "#fef08a",
       transparent: "transparent",
       inherit: "inherit",
+      current: "currentColor",
     },
     fontFamily: {
       sans: ["'Montserrat'", ...fontFamily.sans],

packages/constants/version.ts

@@ -4,3 +4,6 @@ export const VERSION =
   process.env["NODE_ENV"] === "development"
     ? "0.0.0-development"
     : packageJSON.version
+
+export const GIT_REPO_LINK =
+  "https://git.theoludwig.fr/theoludwig/wikipedia-game-solver"

packages/i18n/src/translations/fr-FR.json

@@ -18,6 +18,6 @@
   },
   "home": {
     "title": "Wikipedia Game Solver",
-    "description": "Le jeu Wikipédia implique des joueurs en compétition pour naviguer d'une page Wikipédia à une autre en utilisant uniquement des liens internes."
+    "description": "Le jeu Wikipédia implique des joueurs en compétition pour naviguer d'une page <wikipedia>Wikipedia</wikipedia> à une autre en utilisant uniquement des liens internes."
   }
 }

packages/ui/package.json

@@ -19,6 +19,7 @@
   },
   "dependencies": {
     "@repo/config-tailwind": "workspace:*",
+    "@repo/constants": "workspace:*",
     "@repo/i18n": "workspace:*",
     "@repo/react-hooks": "workspace:*",
     "cva": "catalog:",

packages/ui/src/Footer/Footer.tsx

@@ -1,5 +1,6 @@
 import { useTranslations } from "next-intl"
 
+import { GIT_REPO_LINK } from "@repo/constants"
 import { Link } from "../design/Link/Link"
 
 export interface FooterProps {
@@ -22,9 +23,13 @@ export const Footer: React.FC<FooterProps> = (props) => {
 
       <p>
         Version{" "}
-        <strong className="text-primary dark:text-primary-dark hover:underline">
+        <Link
+          href={`${GIT_REPO_LINK}/releases/tag/v${version}`}
+          target="_blank"
+          isExternal={false}
+        >
           {version}
-        </strong>
+        </Link>
       </p>
     </footer>
   )

packages/ui/src/Header/Locales.tsx

@@ -1,6 +1,7 @@
 "use client"
 
 import { classNames } from "@repo/config-tailwind/classNames"
+import type { Locale } from "@repo/i18n/config"
 import { LOCALES } from "@repo/i18n/config"
 import { usePathname, useRouter } from "@repo/i18n/navigation"
 import { useLocale, useTranslations } from "next-intl"
@@ -8,7 +9,7 @@ import { useLocale, useTranslations } from "next-intl"
 export const Locales: React.FC = () => {
   const router = useRouter()
   const pathname = usePathname()
-  const currentLocale = useLocale()
+  const localeCurrent = useLocale() as Locale
 
   const t = useTranslations()
 
@@ -21,7 +22,7 @@ export const Locales: React.FC = () => {
               key={locale}
               className={classNames("rounded-md p-2", {
                 "border-primary dark:border-primary-dark border":
-                  locale === currentLocale,
+                  locale === localeCurrent,
               })}
             >
               <button

packages/wikipedia-game-solver/src/WikipediaClient.tsx

@@ -1,21 +1,53 @@
 "use client"
 
+import type { Locale } from "@repo/i18n/config"
 import { Button } from "@repo/ui/design/Button"
+import { Link } from "@repo/ui/design/Link"
+import { Typography } from "@repo/ui/design/Typography"
+import { useLocale } from "next-intl"
 import { useState } from "react"
-
+import {
-const wait = async (ms: number): Promise<void> => {
+  fromLocaleToWikipediaLocale,
-  return await new Promise((resolve) => {
+  getWikipediaLink,
-    setTimeout(resolve, ms)
+  getWikipediaPageInternalLinks,
-  })
+} from "./wikipedia-api"
-}
 
 export const WikipediaClient: React.FC = () => {
   const [isLoading, setIsLoading] = useState(false)
 
+  const localeCurrent = useLocale() as Locale
+  const localeWikipedia = fromLocaleToWikipediaLocale(localeCurrent)
+
   const handleClick: React.MouseEventHandler<HTMLButtonElement> = async () => {
-    console.log("clicked")
     setIsLoading(true)
-    await wait(2_000)
+    const fromPageInput = "Linux"
+    const toPageInput = "Node.js"
+    console.log({
+      fromPageInput,
+      toPageInput,
+    })
+    const [fromPageWikipediaLinks, toPageWikipediaLinks] = await Promise.all([
+      getWikipediaPageInternalLinks({
+        title: fromPageInput,
+        locale: localeWikipedia,
+      }),
+      getWikipediaPageInternalLinks({
+        title: toPageInput,
+        locale: localeWikipedia,
+      }),
+    ])
+    console.log({
+      fromPageWikipediaLinks,
+      toPageWikipediaLinks,
+    })
+    // const deepInternalLinks = await getDeepWikipediaPageInternalLinks({
+    //   locale: localeWikipedia,
+    //   data: {
+    //     [fromPageWikipediaLinks.title]: fromPageWikipediaLinks,
+    //     [toPageWikipediaLinks.title]: toPageWikipediaLinks,
+    //   },
+    // })
+    // console.log(deepInternalLinks)
     setIsLoading(false)
   }
 
@@ -24,15 +56,12 @@ export const WikipediaClient: React.FC = () => {
       <Button onClick={handleClick} isLoading={isLoading} className="w-36">
         Wikipedia
       </Button>
-
+      <Typography variant="text1" as="p">
-      <Button
+        <span>Using: </span>
-        onClick={handleClick}
+        <Link href={getWikipediaLink(localeWikipedia)} target="_blank">
-        variant="outline"
+          {getWikipediaLink(localeWikipedia).replace("https://", "")}
-        isLoading={isLoading}
+        </Link>
-        className="w-36"
+      </Typography>
-      >
-        Wikipedia
-      </Button>
     </section>
   )
 }

packages/wikipedia-game-solver/src/wikipedia-api.ts

@@ -1,28 +1,178 @@
+import type { Locale } from "@repo/i18n/config"
+
 export const sum = (a: number, b: number): number => {
   return a + b
 }
 
-export const wikipediaAPIBaseURL = new URL("https://en.wikipedia.org/w/api.php")
+/**
+ * @see https://www.mediawiki.org/wiki/Wikimedia_REST_API#Terms_and_conditions
+ * To avoid impacting other API users, limit your clients to no more than 200 requests/sec to this API overall. Many entry points additionally specify and enforce more restrictive rate limits (HTTP 429 error).
+ */
 
-// const getWikipediaPageLinks = async (title: string): Promise<string[]> => {
+export const WIKIPEDIA_LOCALES = ["en", "fr"] as const
-//   const url = new URL(wikipediaAPIBaseURL)
+export type WikipediaLocale = (typeof WIKIPEDIA_LOCALES)[number]
-//   url.searchParams.append("action", "query")
-//   url.searchParams.append("titles", title)
-//   url.searchParams.append("prop", "links")
-//   url.searchParams.append("pllimit", "max")
-//   url.searchParams.append("format", "json")
-//   url.searchParams.append("origin", "*")
-//   const response = await fetch(url, {
-//     method: "GET",
-//   })
-//   if (!response.ok) {
-//     throw new Error(response.statusText)
-//   }
-//   const json = await response.json()
-//   return json
-// }
 
-// const links = await getWikipediaPageLinks("France")
+const WIKIPEDIA_LOCALES_MAP: Record<Locale, WikipediaLocale> = {
-// const links2 = await getWikipediaPageLinks("Frddgdgdgance")
+  "en-US": "en",
-// console.log("links.length", links)
+  "fr-FR": "fr",
-// console.log("links.length", links2)
+}
+
+export const fromLocaleToWikipediaLocale = (
+  locale: Locale,
+): WikipediaLocale => {
+  return WIKIPEDIA_LOCALES_MAP[locale]
+}
+
+export const getWikipediaLink = (locale: WikipediaLocale): string => {
+  return `https://${locale}.wikipedia.org`
+}
+
+interface WikipediaQueryLinksResponse {
+  continue?: {
+    plcontinue: string
+    continue: string
+  }
+  query: {
+    pages: {
+      [key: string]: {
+        pageid: number
+        ns: number
+        title: string
+        links: [
+          {
+            ns: number
+            title: string
+          },
+        ]
+      }
+    }
+  }
+  limits: {
+    links: number
+  }
+}
+
+interface GetWikipediaPageInternalLinksInput {
+  title: string
+  locale: WikipediaLocale
+}
+
+interface GetWikipediaPageInternalLinksOutput {
+  /**
+   * Title of the Wikipedia page.
+   */
+  title: string
+
+  /**
+   * Page id is unique for each page on Wikipedia, can be used to link to the page.
+   * @example `https://${locale}.wikipedia.org/?curid=${pageId}`
+   */
+  pageId: number
+
+  /**
+   * List of internal links on the Wikipedia page.
+   */
+  links: string[]
+}
+
+/**
+ * Get internal links from a Wikipedia page.
+ * @param input
+ * @returns
+ */
+export const getWikipediaPageInternalLinks = async (
+  input: GetWikipediaPageInternalLinksInput,
+): Promise<GetWikipediaPageInternalLinksOutput> => {
+  const links: string[] = []
+  let title = input.title
+  let pageId = 0
+  let plcontinue: string | null = null
+
+  const fetchLinks = async (): Promise<WikipediaQueryLinksResponse> => {
+    const url = new URL("/w/api.php", getWikipediaLink(input.locale))
+    url.searchParams.append("action", "query")
+    url.searchParams.append("titles", title)
+    url.searchParams.append("prop", "links")
+    url.searchParams.append("pllimit", "max")
+    url.searchParams.append("format", "json")
+    url.searchParams.append("origin", "*")
+    if (plcontinue != null) {
+      url.searchParams.set("plcontinue", plcontinue)
+    }
+    const response = await fetch(url, {
+      method: "GET",
+    })
+    if (!response.ok) {
+      throw new Error(response.statusText)
+    }
+    const json = (await response.json()) as WikipediaQueryLinksResponse
+    return json
+  }
+
+  do {
+    try {
+      const response = await fetchLinks()
+      plcontinue = response?.continue?.plcontinue ?? null
+      const pages = Object.keys(response.query.pages)
+      const page = pages[0] ?? ""
+      if (page === "-1" || page === "") {
+        break
+      }
+      const pageData = response.query.pages[page]
+      if (pageData == null) {
+        break
+      }
+      title = pageData.title
+      pageId = pageData.pageid
+      links.push(
+        ...pageData.links.map((link) => {
+          return link.title
+        }),
+      )
+    } catch {
+      break
+    }
+  } while (plcontinue != null)
+
+  return {
+    title,
+    pageId,
+    links,
+  }
+}
+
+export interface WikipediaPagesInternalLinks {
+  [key: string]: GetWikipediaPageInternalLinksOutput
+}
+
+export interface GetDeepWikipediaPageInternalLinksInput {
+  locale: WikipediaLocale
+  data: WikipediaPagesInternalLinks
+}
+
+export const getDeepWikipediaPageInternalLinks = async (
+  input: GetDeepWikipediaPageInternalLinksInput,
+): Promise<WikipediaPagesInternalLinks> => {
+  const pagesTitles = Object.keys(input.data)
+  await Promise.all(
+    pagesTitles.map(async (pageTitle) => {
+      const links = input.data[pageTitle]?.links ?? []
+      await Promise.all(
+        links.map(async (pageTitleLink) => {
+          if (pageTitleLink in input.data) {
+            return
+          }
+          input.data[pageTitleLink] = await getWikipediaPageInternalLinks({
+            locale: input.locale,
+            title: pageTitleLink,
+          })
+          await getDeepWikipediaPageInternalLinks({
+            locale: input.locale,
+            data: input.data,
+          })
+        }),
+      )
+    }),
+  )
+  return input.data
+}

pnpm-lock.yaml

@@ -547,6 +547,9 @@ importers:
       '@repo/config-tailwind':
         specifier: workspace:*
         version: link:../config-tailwind
+      '@repo/constants':
+        specifier: workspace:*
+        version: link:../constants
       '@repo/i18n':
         specifier: workspace:*
         version: link:../i18n