Implement MSC3575: Sliding Sync (#8328)

* Add labs flag for sliding sync; add sliding_sync_proxy_url to config.json * Disable the labs toggle if sliding_sync_proxy_url is not set * Do validation checks on the sliding sync proxy URL before enabling it in Labs * Enable sliding sync and add SlidingSyncManager * Get room subscriptions working * Hijack renderSublists in sliding sync mode * Add support for sorting alphabetically/recency and room name filters * Filter out tombstoned rooms; start adding show more logic list ranges update but the UI doesn't * update the UI when the list is updated * bugfix: make sure the list sorts numerically * Get invites transitioning correctly * Force enable sliding sync and labs for now * Linting * Disable spotlight search * Initial cypress plugins for Sliding Sync Proxy * Use --rm when running Synapse in Docker for Cypress tests * Update src/MatrixClientPeg.ts Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/components/views/rooms/RoomSublist.tsx Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/settings/controllers/SlidingSyncController.ts Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/components/views/rooms/RoomSublist.tsx Co-authored-by: Travis Ralston <travisr@matrix.org> * WIP add room searching to spotlight search * Only read sliding sync results when there is a result, else use the local cache * Use feature_sliding_sync not slidingSync * Some review comments * More review comments * Use RoomViewStore to set room subscriptions * Comment why any * Update src/components/views/rooms/RoomSublist.tsx Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> * Fix cypress docker abstraction * Iterate sliding sync proxy support * Stash mostly functional test * Update sliding sync proxy image * i18n * Add support for spaces; use list ID -> index mappings - Mappings are more reusable and easier to understand than racing for index positions. - Register for all spaces immediately on startup. * When the active space is updated, update the list registration * Set spaces filter in the correct place * Skeleton placeholder whilst loading the space * Filter out spaces from the room list * Use the new txn_id promises * Ensure we actually resolve list registrations * Fix matrix-org/sliding-sync#30: don't show tombstoned search results * Remove unused imports * Add SYNCV3_SECRET to proxy to ensure it starts up; correct aliases for SS test * Add another basic sliding sync e2e test * Unbreak netlify * Add more logging for debugging duplicate rooms * If sliding sync is enabled, always use the rooms result even if it's empty * Drop-in copy of RoomListStore for sliding sync * Remove conditionals from RoomListStore - we have SlidingRoomListStore now * WIP SlidingRoomListStore * Add most sliding sync logic to SlidingRoomListStore Still lots of logic in RoomSublist. Broken things: - Join count is wrong completely. - No skeleton placeholder when switching spaces. * Migrate joined count to SS RLS * Reinstate the skeleton UI when the list is loading * linting * Add support for sticky rooms based on the currently active room * Add a bunch of passing SS E2E tests; some WIP * Unbreak build from git merge * Suppress unread indicators in sliding sync mode * Add regression test for https://github.com/matrix-org/sliding-sync/issues/28 * Add invite test flows; show the invite list The refactor to SS RLS removed the invite list entirely. * Remove show more click as it wasn't the bug * Linting and i18n * only enable SS by default on netlify * Jest fixes; merge conflict fixes; remove debug logging; use right sort enum values * Actually fix jest tests * Add support for favourites and low priority * Bump sliding sync version * Update sliding sync labs to be user configurable * delint * To disable SS or change proxy URL the user has to log out * Review comments * Linting * Apply suggestions from code review Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/stores/room-list/SlidingRoomListStore.ts Co-authored-by: Travis Ralston <travisr@matrix.org> * Review comments * Add issue link for TODO markers * Linting * Apply suggestions from code review Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> * More review comments * More review comments * stricter types Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> Co-authored-by: Travis Ralston <travisr@matrix.org>
2022-09-07 15:42:39 +00:00
parent 5bdae150fa
commit a215027c6b
25 changed files with 1632 additions and 51 deletions
@@ -17,6 +17,7 @@ limitations under the License.
 /// <reference types="cypress" />

 import * as os from "os";
+import * as crypto from "crypto";
 import * as childProcess from "child_process";
 import * as fse from "fs-extra";

@@ -25,28 +26,32 @@ import PluginConfigOptions = Cypress.PluginConfigOptions;

 // A cypress plugin to run docker commands

-export function dockerRun(args: {
+export function dockerRun(opts: {
    image: string;
    containerName: string;
    params?: string[];
+    cmd?: string;
 }): Promise<string> {
    const userInfo = os.userInfo();
-    const params = args.params ?? [];
+    const params = opts.params ?? [];

-    if (userInfo.uid >= 0) {
+    if (params?.includes("-v") && userInfo.uid >= 0) {
        // On *nix we run the docker container as our uid:gid otherwise cleaning it up its media_store can be difficult
        params.push("-u", `${userInfo.uid}:${userInfo.gid}`);
    }

+    const args = [
+        "run",
+        "--name", `${opts.containerName}-${crypto.randomBytes(4).toString("hex")}`,
+        "-d",
+        ...params,
+        opts.image,
+    ];
+
+    if (opts.cmd) args.push(opts.cmd);
+
    return new Promise<string>((resolve, reject) => {
-        childProcess.execFile('docker', [
-            "run",
-            "--name", args.containerName,
-            "-d",
-            ...params,
-            args.image,
-            "run",
-        ], (err, stdout) => {
+        childProcess.execFile("docker", args, (err, stdout) => {
            if (err) reject(err);
            resolve(stdout.trim());
        });
@@ -122,6 +127,21 @@ export function dockerRm(args: {
    });
 }

+export function dockerIp(args: {
+    containerId: string;
+}): Promise<string> {
+    return new Promise<string>((resolve, reject) => {
+        childProcess.execFile('docker', [
+            "inspect",
+            "-f", "{{ .NetworkSettings.IPAddress }}",
+            args.containerId,
+        ], (err, stdout) => {
+            if (err) reject(err);
+            else resolve(stdout.trim());
+        });
+    });
+}
+
 /**
 * @type {Cypress.PluginConfig}
 */
@@ -132,5 +152,6 @@ export function docker(on: PluginEvents, config: PluginConfigOptions) {
        dockerLogs,
        dockerStop,
        dockerRm,
+        dockerIp,
    });
 }
@@ -20,6 +20,7 @@ import PluginEvents = Cypress.PluginEvents;
 import PluginConfigOptions = Cypress.PluginConfigOptions;
 import { performance } from "./performance";
 import { synapseDocker } from "./synapsedocker";
+import { slidingSyncProxyDocker } from "./sliding-sync";
 import { webserver } from "./webserver";
 import { docker } from "./docker";
 import { log } from "./log";
@@ -31,6 +32,7 @@ export default function(on: PluginEvents, config: PluginConfigOptions) {
    docker(on, config);
    performance(on, config);
    synapseDocker(on, config);
+    slidingSyncProxyDocker(on, config);
    webserver(on, config);
    log(on, config);
 }
@@ -0,0 +1,128 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import PluginEvents = Cypress.PluginEvents;
+import PluginConfigOptions = Cypress.PluginConfigOptions;
+import { dockerExec, dockerIp, dockerRun, dockerStop } from "../docker";
+import { getFreePort } from "../utils/port";
+import { SynapseInstance } from "../synapsedocker";
+
+// A cypress plugins to add command to start & stop https://github.com/matrix-org/sliding-sync
+
+export interface ProxyInstance {
+    containerId: string;
+    postgresId: string;
+    port: number;
+}
+
+const instances = new Map<string, ProxyInstance>();
+
+const PG_PASSWORD = "p4S5w0rD";
+
+async function proxyStart(synapse: SynapseInstance): Promise<ProxyInstance> {
+    console.log(new Date(), "Starting sliding sync proxy...");
+
+    const postgresId = await dockerRun({
+        image: "postgres",
+        containerName: "react-sdk-cypress-sliding-sync-postgres",
+        params: [
+            "--rm",
+            "-e", `POSTGRES_PASSWORD=${PG_PASSWORD}`,
+        ],
+    });
+
+    const postgresIp = await dockerIp({ containerId: postgresId });
+    const synapseIp = await dockerIp({ containerId: synapse.synapseId });
+    console.log(new Date(), "postgres container up");
+
+    const waitTimeMillis = 30000;
+    const startTime = new Date().getTime();
+    let lastErr: Error;
+    while ((new Date().getTime() - startTime) < waitTimeMillis) {
+        try {
+            await dockerExec({
+                containerId: postgresId,
+                params: [
+                    "pg_isready",
+                    "-U", "postgres",
+                ],
+            });
+            lastErr = null;
+            break;
+        } catch (err) {
+            console.log("pg_isready: failed");
+            lastErr = err;
+        }
+    }
+    if (lastErr) {
+        console.log("rethrowing");
+        throw lastErr;
+    }
+
+    const port = await getFreePort();
+    console.log(new Date(), "starting proxy container...");
+    const containerId = await dockerRun({
+        image: "ghcr.io/matrix-org/sliding-sync-proxy:v0.4.0",
+        containerName: "react-sdk-cypress-sliding-sync-proxy",
+        params: [
+            "--rm",
+            "-p", `${port}:8008/tcp`,
+            "-e", "SYNCV3_SECRET=bwahahaha",
+            "-e", `SYNCV3_SERVER=http://${synapseIp}:8008`,
+            "-e", `SYNCV3_DB=user=postgres dbname=postgres password=${PG_PASSWORD} host=${postgresIp} sslmode=disable`,
+        ],
+    });
+    console.log(new Date(), "started!");
+
+    const instance: ProxyInstance = { containerId, postgresId, port };
+    instances.set(containerId, instance);
+    return instance;
+}
+
+async function proxyStop(instance: ProxyInstance): Promise<void> {
+    await dockerStop({
+        containerId: instance.containerId,
+    });
+    await dockerStop({
+        containerId: instance.postgresId,
+    });
+
+    instances.delete(instance.containerId);
+
+    console.log(new Date(), "Stopped sliding sync proxy.");
+    // cypress deliberately fails if you return 'undefined', so
+    // return null to signal all is well, and we've handled the task.
+    return null;
+}
+
+/**
+ * @type {Cypress.PluginConfig}
+ */
+export function slidingSyncProxyDocker(on: PluginEvents, config: PluginConfigOptions) {
+    on("task", {
+        proxyStart,
+        proxyStop,
+    });
+
+    on("after:spec", async (spec) => {
+        for (const instance of instances.values()) {
+            console.warn(`Cleaning up proxy on port ${instance.port} after ${spec.name}`);
+            await proxyStop(instance);
+        }
+    });
+}
@@ -101,12 +101,13 @@ async function synapseStart(template: string): Promise<SynapseInstance> {

    const synapseId = await dockerRun({
        image: "matrixdotorg/synapse:develop",
-        containerName: `react-sdk-cypress-synapse-${crypto.randomBytes(4).toString("hex")}`,
+        containerName: `react-sdk-cypress-synapse`,
        params: [
            "--rm",
            "-v", `${synCfg.configDir}:/data`,
            "-p", `${synCfg.port}:8008/tcp`,
        ],
+        cmd: "run",
    });

    console.log(`Started synapse with id ${synapseId} on port ${synCfg.port}.`);