Rewrite copy-res i18n build as a webpack plugin (#32664)

* Replace copy-res with a webpack plugin

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Specify writeToDisk=true for webpack-dev-server

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Use async fs methods

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

apps/web/I18nWebpackPlugin.ts

@@ -0,0 +1,123 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import webpack from "webpack";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import _ from "lodash";
+import { type Translations } from "matrix-web-i18n";
+
+interface Options {
+    // Path to the strings for the application, will be used to deduce what languages are supported
+    stringsPath: string;
+    // Additional paths to strings which will be merged into the application's own strings for supported languages
+    additionalStringsPaths?: string[];
+}
+
+async function exists(path: string): Promise<boolean> {
+    try {
+        await fs.access(path);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+export class I18nWebpackPlugin {
+    private readonly options: Options;
+
+    public constructor(options: Options) {
+        this.options = options;
+    }
+
+    public apply(compiler: webpack.Compiler): void {
+        const { RawSource } = compiler.webpack.sources;
+
+        compiler.hooks.thisCompilation.tap("I18nWebpackPlugin", (compilation) => {
+            compilation.hooks.processAssets.tapPromise(
+                {
+                    name: "I18nWebpackPlugin",
+                    stage: webpack.Compilation.PROCESS_ASSETS_STAGE_ADDITIONS,
+                },
+                async () => {
+                    const paths = [this.options.stringsPath, ...(this.options.additionalStringsPaths ?? [])].map((p) =>
+                        path.resolve(compiler.context, p),
+                    );
+
+                    const logger = compilation.getLogger("I18nWebpackPlugin");
+
+                    for (const p of paths) {
+                        compilation.contextDependencies.add(p);
+
+                        if (!(await exists(p))) {
+                            compilation.errors.push(
+                                new webpack.WebpackError(`I18nWebpackPlugin: strings path not found: ${p}`),
+                            );
+                            return;
+                        }
+                    }
+
+                    const primaryPath = paths[0];
+                    const includeLangs = [...new Set([...(await fs.readdir(primaryPath))])]
+                        .filter((fn) => fn.endsWith(".json"))
+                        .map((f) => f.slice(0, -5));
+
+                    const langFileMap: Record<string, string> = {};
+
+                    for (const lang of includeLangs) {
+                        let translations: Translations = {};
+                        for (const p of paths) {
+                            const f = path.join(p, lang + ".json");
+
+                            if (await exists(f)) {
+                                try {
+                                    const content = await fs.readFile(f, "utf-8");
+                                    translations = _.merge(translations, JSON.parse(content));
+                                    compilation.fileDependencies.add(f);
+                                } catch (e) {
+                                    compilation.errors.push(
+                                        new webpack.WebpackError(
+                                            `I18nWebpackPlugin: Failed to read or parse ${f}: ${e}`,
+                                        ),
+                                    );
+                                }
+                            }
+                        }
+
+                        const json = JSON.stringify(translations, null, 4);
+                        const jsonBuffer = Buffer.from(json);
+                        const digest = compiler.webpack.util
+                            .createHash("xxhash64")
+                            .update(jsonBuffer)
+                            .digest("hex")
+                            .slice(0, 7);
+                        const filename = `${lang}.${digest}.json`;
+
+                        compilation.emitAsset(`i18n/${filename}`, new RawSource(jsonBuffer));
+                        langFileMap[lang] = filename;
+                        logger.debug(`Generated language file: ${filename}`);
+                    }
+
+                    // Generate languages.json
+                    const languages: Record<string, string> = {};
+                    includeLangs.forEach((lang) => {
+                        const normalizedLanguage = lang.toLowerCase().replace("_", "-");
+                        const languageParts = normalizedLanguage.split("-");
+                        if (languageParts.length == 2 && languageParts[0] == languageParts[1]) {
+                            languages[languageParts[0]] = langFileMap[lang];
+                        } else {
+                            languages[normalizedLanguage] = langFileMap[lang];
+                        }
+                    });
+
+                    compilation.emitAsset("i18n/languages.json", new RawSource(JSON.stringify(languages, null, 4)));
+                    logger.info("Generated languages.json and language files");
+                },
+            );
+        });
+    }
+}

apps/web/package.json

@@ -159,7 +159,6 @@
         "@types/jitsi-meet": "^2.0.2",
         "@types/jsrsasign": "^10.5.4",
         "@types/lodash": "^4.14.168",
-        "@types/minimist": "^1.2.5",
         "@types/modernizr": "^3.5.3",
         "@types/node": "22",
         "@types/pako": "^2.0.0",
@@ -180,7 +179,6 @@
         "babel-loader": "^10.0.0",
         "babel-plugin-jsx-remove-data-test-id": "^3.0.0",
         "blob-polyfill": "^9.0.0",
-        "chokidar": "^5.0.0",
         "copy-webpack-plugin": "^13.0.0",
         "css-loader": "^7.0.0",
         "css-minimizer-webpack-plugin": "^7.0.0",
@@ -211,7 +209,6 @@
         "jsqr": "^1.4.0",
         "matrix-web-i18n": "catalog:",
         "mini-css-extract-plugin": "2.10.0",
-        "minimist": "^1.2.6",
         "modernizr": "^3.12.0",
         "postcss": "8.5.6",
         "postcss-easings": "4.0.0",

apps/web/project.json

@@ -2,16 +2,6 @@
     "$schema": "../../node_modules/nx/schemas/project-schema.json",
     "projectType": "application",
     "targets": {
-        "prebuild:i18n": {
-            "cache": true,
-            "command": "node scripts/copy-res.ts",
-            "inputs": [
-                "{projectRoot}/src/i18n/strings/*.json",
-                "{workspaceRoot}/packages/shared-components/src/i18n/strings/*.json"
-            ],
-            "outputs": ["{projectRoot}/webapp/i18n/"],
-            "options": { "cwd": "apps/web" }
-        },
         "prebuild:module_system": {
             "cache": true,
             "command": "node module_system/scripts/install.ts",
@@ -34,14 +24,9 @@
             "outputs": ["{projectRoot}/webapp"],
             "options": { "cwd": "apps/web" }
         },
-        "start:i18n": {
-            "command": "node scripts/copy-res.ts -w",
-            "continuous": true,
-            "options": { "cwd": "apps/web" }
-        },
         "start": {
             "command": "webpack-dev-server --disable-interpret --output-path webapp --output-filename=bundles/_dev_/[name].js --output-chunk-filename=bundles/_dev_/[name].js --mode development",
-            "dependsOn": ["prebuild:module_system", "prebuild:rethemendex", "start:i18n", "^start"],
+            "dependsOn": ["prebuild:module_system", "prebuild:rethemendex", "^start"],
             "continuous": true,
             "options": { "cwd": "apps/web" }
         },

apps/web/scripts/copy-res.ts

@@ -1,152 +0,0 @@
-#!/usr/bin/env node
-
-// copies the resources into the webapp directory.
-
-import parseArgs from "minimist";
-import * as chokidar from "chokidar";
-import * as fs from "node:fs";
-import _ from "lodash";
-import webpack from "webpack";
-import { type Translations } from "matrix-web-i18n";
-
-const EW_I18N_BASE_PATH = "src/i18n/strings/";
-const SC_I18N_BASE_PATH = "../../packages/shared-components/src/i18n/strings/";
-
-const INCLUDE_LANGS = [...new Set([...fs.readdirSync(EW_I18N_BASE_PATH)])]
-    .filter((fn) => fn.endsWith(".json"))
-    .map((f) => f.slice(0, -5));
-
-const argv = parseArgs(process.argv.slice(2), {});
-
-const watch = argv.w;
-const verbose = argv.v;
-
-function errCheck(err: unknown): void {
-    if (err) {
-        console.error(err instanceof Error ? err.message : err);
-        process.exit(1);
-    }
-}
-
-const I18N_DEST = "webapp/i18n/";
-// Check if webapp/i18n exists
-if (!fs.existsSync(I18N_DEST)) {
-    fs.mkdirSync(I18N_DEST, { recursive: true });
-}
-
-const logWatch = (path: string): void => {
-    if (verbose) {
-        console.log(`Watching: ${path}`);
-    }
-};
-
-/*
- * Make a JSON language file for the given language by merging all translations
- * into a single file (ie. element-web and shared-components).
- * Returns the filename (including hash) and JSON content.
- */
-function prepareLangFile(lang: string): [filename: string, json: string] {
-    const ewTranslationsPath = EW_I18N_BASE_PATH + lang + ".json";
-    const scTranslationsPath = SC_I18N_BASE_PATH + lang + ".json";
-
-    let translations: Translations = {};
-    [ewTranslationsPath, scTranslationsPath].forEach(function (f) {
-        if (fs.existsSync(f)) {
-            try {
-                translations = _.merge(translations, JSON.parse(fs.readFileSync(f).toString()));
-            } catch (e) {
-                console.error("Failed: " + f, e);
-                throw e;
-            }
-        }
-    });
-
-    const json = JSON.stringify(translations, null, 4);
-    const jsonBuffer = Buffer.from(json);
-    const digest = webpack.util.createHash("xxhash64").update(jsonBuffer).digest("hex").slice(0, 7);
-    const filename = `${lang}.${digest}.json`;
-
-    return [filename, json];
-}
-
-function genLangFile(dest: string, filename: string, json: string): void {
-    fs.writeFileSync(dest + filename, json);
-    if (verbose) {
-        console.log("Generated language file: " + filename);
-    }
-}
-
-function genLangList(langFileMap: Record<string, string>): void {
-    const languages: Record<string, string> = {};
-    INCLUDE_LANGS.forEach(function (lang) {
-        const normalizedLanguage = lang.toLowerCase().replace("_", "-");
-        const languageParts = normalizedLanguage.split("-");
-        if (languageParts.length == 2 && languageParts[0] == languageParts[1]) {
-            languages[languageParts[0]] = langFileMap[lang];
-        } else {
-            languages[normalizedLanguage] = langFileMap[lang];
-        }
-    });
-    fs.writeFile("webapp/i18n/languages.json", JSON.stringify(languages, null, 4), function (err) {
-        if (err) {
-            console.error("Copy Error occured: " + err.message);
-            throw new Error("Failed to generate languages.json");
-        }
-    });
-    if (verbose) {
-        console.log("Generated languages.json");
-    }
-}
-
-/*
- * watch the input files for a given language,
- * regenerate the file, adding its content-hashed filename to langFileMap
- * and regenerating languages.json with the new filename
- */
-function watchLanguage(lang: string, dest: string, langFileMap: Record<string, string>): void {
-    const ewTranslationsPath = EW_I18N_BASE_PATH + lang + ".json";
-    const scTranslationsPath = SC_I18N_BASE_PATH + lang + ".json";
-
-    // XXX: Use a debounce because for some reason if we read the language
-    // file immediately after the FS event is received, the file contents
-    // appears empty. Possibly https://github.com/nodejs/node/issues/6112
-    let makeLangDebouncer: ReturnType<typeof setTimeout>;
-    const makeLang = (): void => {
-        if (makeLangDebouncer) {
-            clearTimeout(makeLangDebouncer);
-        }
-        makeLangDebouncer = setTimeout(() => {
-            const [filename, json] = prepareLangFile(lang);
-            genLangFile(dest, filename, json);
-            langFileMap[lang] = filename;
-            genLangList(langFileMap);
-        }, 500);
-    };
-
-    [ewTranslationsPath, scTranslationsPath].forEach(function (f) {
-        chokidar
-            .watch(f, { ignoreInitial: true })
-            .on("ready", () => {
-                logWatch(f);
-            })
-            .on("add", makeLang)
-            .on("change", makeLang)
-            .on("error", errCheck);
-    });
-}
-
-// language resources
-const I18N_FILENAME_MAP = INCLUDE_LANGS.reduce<Record<string, string>>((m, l) => {
-    const [filename, json] = prepareLangFile(l);
-    if (!watch) {
-        genLangFile(I18N_DEST, filename, json);
-    }
-    m[l] = filename;
-    return m;
-}, {});
-
-if (watch) {
-    INCLUDE_LANGS.forEach((l) => watchLanguage(l, I18N_DEST, I18N_FILENAME_MAP));
-} else {
-    genLangList(I18N_FILENAME_MAP);
-}

apps/web/webpack.config.ts

@@ -29,6 +29,7 @@ import postcssEasings from "postcss-easings";
 
 import pkgJson from "./package.json" with { type: "json" };
 import componentsJson from "./components.json" with { type: "json" };
+import { I18nWebpackPlugin } from "./I18nWebpackPlugin.ts";
 import type { sentryWebpackPlugin as sentryWebpackPluginType } from "@sentry/webpack-plugin/webpack5";
 
 // Environment variables
@@ -618,6 +619,11 @@ export default (env: string, argv: Record<string, any>): webpack.Configuration =
         plugins: [
             ...moduleReplacementPlugins,
 
+            new I18nWebpackPlugin({
+                stringsPath: "src/i18n/strings/",
+                additionalStringsPaths: ["../../packages/shared-components/src/i18n/strings/"],
+            }),
+
             // This exports our CSS using the splitChunks and loaders above.
             new MiniCssExtractPlugin({
                 filename: "bundles/[fullhash]/[name].css",
@@ -793,6 +799,7 @@ export default (env: string, argv: Record<string, any>): webpack.Configuration =
                 // Only output errors, warnings, or new compilations.
                 // This hides the massive list of modules.
                 stats: "minimal",
+                writeToDisk: true,
             },
 
             // Enable Hot Module Replacement without page refresh as a fallback in

pnpm-lock.yaml

@@ -495,9 +495,6 @@ importers:
       '@types/lodash':
         specifier: ^4.14.168
         version: 4.17.23
-      '@types/minimist':
-        specifier: ^1.2.5
-        version: 1.2.5
       '@types/modernizr':
         specifier: ^3.5.3
         version: 3.5.6
@@ -558,9 +555,6 @@ importers:
       blob-polyfill:
         specifier: ^9.0.0
         version: 9.0.20240710
-      chokidar:
-        specifier: ^5.0.0
-        version: 5.0.0
       copy-webpack-plugin:
         specifier: ^13.0.0
         version: 13.0.1(webpack@5.105.2)
@@ -651,9 +645,6 @@ importers:
       mini-css-extract-plugin:
         specifier: 2.10.0
         version: 2.10.0(webpack@5.105.2)
-      minimist:
-        specifier: ^1.2.6
-        version: 1.2.8
       modernizr:
         specifier: ^3.12.0
         version: 3.13.1
@@ -4267,9 +4258,6 @@ packages:
   '@types/mime@1.3.5':
     resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==}
 
-  '@types/minimist@1.2.5':
-    resolution: {integrity: sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==}
-
   '@types/modernizr@3.5.6':
     resolution: {integrity: sha512-yslwR0zZ3zAT1qXcCPxIcD23CZ6W6nKsl6JufSJHAmdwOBuYwCVJkaMsEo9yzxGV7ATfoX8S+RgtnajOEtKxYA==}
 
@@ -5455,10 +5443,6 @@ packages:
     resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
     engines: {node: '>= 8.10.0'}
 
-  chokidar@5.0.0:
-    resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==}
-    engines: {node: '>= 20.19.0'}
-
   chownr@1.1.4:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
 
@@ -9256,10 +9240,6 @@ packages:
     resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
     engines: {node: '>=8.10.0'}
 
-  readdirp@5.0.0:
-    resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==}
-    engines: {node: '>= 20.19.0'}
-
   recast@0.23.11:
     resolution: {integrity: sha512-YTUo+Flmw4ZXiWfQKGcwwc11KnoRAYgzAE2E7mXKCjSviTKShtxBsN6YUUBB2gtaBzKzeKunxhUwNHQuRryhWA==}
     engines: {node: '>= 4'}
@@ -14506,8 +14486,6 @@ snapshots:
 
   '@types/mime@1.3.5': {}
 
-  '@types/minimist@1.2.5': {}
-
   '@types/modernizr@3.5.6': {}
 
   '@types/node@18.19.130':
@@ -15919,10 +15897,6 @@ snapshots:
     optionalDependencies:
       fsevents: 2.3.3
 
-  chokidar@5.0.0:
-    dependencies:
-      readdirp: 5.0.0
-
   chownr@1.1.4: {}
 
   chrome-trace-event@1.0.4: {}
@@ -20425,8 +20399,6 @@ snapshots:
     dependencies:
       picomatch: 2.3.1
 
-  readdirp@5.0.0: {}
-
   recast@0.23.11:
     dependencies:
       ast-types: 0.16.1