init with v1.7.12

scripts/build-jitsi.js

@@ -0,0 +1,31 @@
+// This is a JS script so that the directory is created in-process on Windows.
+// If the script isn't run in-process, there's a risk of it racing or never running
+// due to file associations in Windows.
+// Sorry.
+
+const fs = require("fs");
+const path = require("path");
+const mkdirp = require("mkdirp");
+const fetch = require("node-fetch");
+const ProxyAgent = require("simple-proxy-agent");
+
+console.log("Making webapp directory");
+mkdirp.sync("webapp");
+
+// curl -s https://jitsi.riot.im/libs/external_api.min.js > ./webapp/jitsi_external_api.min.js
+console.log("Downloading Jitsi script");
+const fname = path.join("webapp", "jitsi_external_api.min.js");
+
+const options = {};
+if (process.env.HTTPS_PROXY) {
+   options.agent = new ProxyAgent(process.env.HTTPS_PROXY, {tunnel: true});
+}
+
+fetch("https://jitsi.riot.im/libs/external_api.min.js", options).then(res => {
+   const stream = fs.createWriteStream(fname);
+   return new Promise((resolve, reject) => {
+       res.body.pipe(stream);
+       res.body.on('error', err => reject(err));
+       res.body.on('finish', () => resolve());
+   });
+}).then(() => console.log('Done with Jitsi download'));

scripts/check-i18n.pl

@@ -0,0 +1 @@
+../../matrix-react-sdk/scripts/check-i18n.pl

scripts/ci_package.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Runs package.sh setting the version to git hashes of the element-web,
+# react-sdk & js-sdk checkouts, for the case where these dependencies
+# are git checkouts.
+
+set -ex
+
+rm dist/riot-*.tar.gz || true # rm previous artifacts without failing if it doesn't exist
+
+# Since the deps are fetched from git, we can rev-parse
+REACT_SHA=$(cd node_modules/matrix-react-sdk; git rev-parse --short=12 HEAD)
+JSSDK_SHA=$(cd node_modules/matrix-js-sdk; git rev-parse --short=12 HEAD)
+
+VECTOR_SHA=$(git rev-parse --short=12 HEAD) # use the ACTUAL SHA rather than assume develop
+
+CI_PACKAGE=true DIST_VERSION=$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA scripts/package.sh -d

scripts/cleanup.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# script to clean up the deployments made by redeploy.py for vectorwebdev and vectorwebexp
+set -e
+
+# currently live deployment (full path)
+live=`dirname $(readlink -f "$HOME/live")`
+
+# currently live bundle (just the name of the bundle)
+live_bundle=`grep 'script src="bundles' live/index.html | sed -e 's#.*bundles/##' -e 's#/.*##'`
+
+# clean up 'extracted': find things which are older than 7 days, exclude the
+# current live one, and remove
+find $HOME/extracted -mindepth 1 -maxdepth 1 -type d -ctime +7 \
+     \! -path "$live" -exec rm -r {} \;
+
+# clean up 'bundles': ditto
+find $HOME/bundles -mindepth 1 -maxdepth 1 -type d -ctime +7 \
+     \! -name "$live_bundle" -exec rm -r {} \;

scripts/compare-file.js

@@ -0,0 +1,10 @@
+const fs = require("fs");
+
+if (process.argv.length < 4) throw new Error("Missing source and target file arguments");
+
+const sourceFile = fs.readFileSync(process.argv[2], 'utf8');
+const targetFile = fs.readFileSync(process.argv[3], 'utf8');
+
+if (sourceFile !== targetFile) {
+    throw new Error("Files do not match");
+}

scripts/copy-res.js

@@ -0,0 +1,291 @@
+#!/usr/bin/env node
+
+const loaderUtils = require("loader-utils");
+
+// copies the resources into the webapp directory.
+//
+
+// Languages are listed manually so we can choose when to include
+// a translation in the app (because having a translation with only
+// 3 strings translated is just frustrating)
+// This could readily be automated, but it's nice to explicitly
+// control when new languages are available.
+const INCLUDE_LANGS = [
+    {'value': 'bg', 'label': 'Български'},
+    {'value': 'ca', 'label': 'Català'},
+    {'value': 'cs', 'label': 'čeština'},
+    {'value': 'da', 'label': 'Dansk'},
+    {'value': 'de_DE', 'label': 'Deutsch'},
+    {'value': 'el', 'label': 'Ελληνικά'},
+    {'value': 'en_EN', 'label': 'English'},
+    {'value': 'en_US', 'label': 'English (US)'},
+    {'value': 'eo', 'label': 'Esperanto'},
+    {'value': 'es', 'label': 'Español'},
+    {'value': 'et', 'label': 'Eesti'},
+    {'value': 'eu', 'label': 'Euskara'},
+    {'value': 'fi', 'label': 'Suomi'},
+    {'value': 'fr', 'label': 'Français'},
+    {'value': 'gl', 'label': 'Galego'},
+    {'value': 'hi', 'label': 'हिन्दी'},
+    {'value': 'hu', 'label': 'Magyar'},
+    {'value': 'is', 'label': 'íslenska'},
+    {'value': 'it', 'label': 'Italiano'},
+    {'value': 'ja', 'label': '日本語'},
+    {'value': 'jbo', 'label': 'banjubu\'o'},
+    {'value': 'kab', 'label': 'Taqbaylit'},
+    {'value': 'ko', 'label': '한국어'},
+    {'value': 'lt', 'label': 'Lietuvių'},
+    {'value': 'lv', 'label': 'Latviešu'},
+    {'value': 'nb_NO', 'label': 'Norwegian Bokmål'},
+    {'value': 'nl', 'label': 'Nederlands'},
+    {'value': 'nn', 'label': 'Norsk Nynorsk'},
+    {'value': 'pl', 'label': 'Polski'},
+    {'value': 'pt', 'label': 'Português'},
+    {'value': 'pt_BR', 'label': 'Português do Brasil'},
+    {'value': 'ru', 'label': 'Русский'},
+    {'value': 'sk', 'label': 'Slovenčina'},
+    {'value': 'sq', 'label': 'Shqip'},
+    {'value': 'sr', 'label': 'српски'},
+    {'value': 'sv', 'label': 'Svenska'},
+    {'value': 'te', 'label': 'తెలుగు'},
+    {'value': 'th', 'label': 'ไทย'},
+    {'value': 'tr', 'label': 'Türkçe'},
+    {'value': 'uk', 'label': 'українська мова'},
+    {'value': 'vls', 'label': 'West-Vlaams'},
+    {'value': 'zh_Hans', 'label': '简体中文'}, // simplified chinese
+    {'value': 'zh_Hant', 'label': '繁體中文'}, // traditional chinese
+];
+
+// cpx includes globbed parts of the filename in the destination, but excludes
+// common parents. Hence, "res/{a,b}/**": the output will be "dest/a/..." and
+// "dest/b/...".
+const COPY_LIST = [
+    ["res/manifest.json", "webapp"],
+    ["res/sw.js", "webapp"],
+    ["res/welcome.html", "webapp"],
+    ["res/welcome/**", "webapp/welcome"],
+    ["res/themes/**", "webapp/themes"],
+    ["res/vector-icons/**", "webapp/vector-icons"],
+    ["res/decoder-ring/**", "webapp/decoder-ring"],
+    ["node_modules/matrix-react-sdk/res/media/**", "webapp/media"],
+    ["node_modules/olm/olm_legacy.js", "webapp", { directwatch: 1 }],
+    ["./config.json", "webapp", { directwatch: 1 }],
+    ["contribute.json", "webapp"],
+];
+
+const parseArgs = require('minimist');
+const Cpx = require('cpx');
+const chokidar = require('chokidar');
+const fs = require('fs');
+const rimraf = require('rimraf');
+
+const argv = parseArgs(
+    process.argv.slice(2), {}
+);
+
+const watch = argv.w;
+const verbose = argv.v;
+
+function errCheck(err) {
+    if (err) {
+        console.error(err.message);
+        process.exit(1);
+    }
+}
+
+// Check if webapp exists
+if (!fs.existsSync('webapp')) {
+    fs.mkdirSync('webapp');
+}
+// Check if i18n exists
+if (!fs.existsSync('webapp/i18n/')) {
+    fs.mkdirSync('webapp/i18n/');
+}
+
+function next(i, err) {
+    errCheck(err);
+
+    if (i >= COPY_LIST.length) {
+        return;
+    }
+
+    const ent = COPY_LIST[i];
+    const source = ent[0];
+    const dest = ent[1];
+    const opts = ent[2] || {};
+    let cpx = undefined;
+
+    if (!opts.lang) {
+        cpx = new Cpx.Cpx(source, dest);
+    }
+
+    if (verbose && cpx) {
+        cpx.on("copy", (event) => {
+            console.log(`Copied: ${event.srcPath} --> ${event.dstPath}`);
+        });
+        cpx.on("remove", (event) => {
+            console.log(`Removed: ${event.path}`);
+        });
+    }
+
+    const cb = (err) => { next(i + 1, err) };
+
+    if (watch) {
+        if (opts.directwatch) {
+            // cpx -w creates a watcher for the parent of any files specified,
+            // which in the case of config.json is '.', which inevitably takes
+            // ages to crawl. So we create our own watcher on the files
+            // instead.
+            const copy = () => { cpx.copy(errCheck) };
+            chokidar.watch(source)
+                .on('add', copy)
+                .on('change', copy)
+                .on('ready', cb)
+                .on('error', errCheck);
+        } else {
+            cpx.on('watch-ready', cb);
+            cpx.on("watch-error", cb);
+            cpx.watch();
+        }
+    } else {
+        cpx.copy(cb);
+    }
+}
+
+function genLangFile(lang, dest) {
+    const reactSdkFile = 'node_modules/matrix-react-sdk/src/i18n/strings/' + lang + '.json';
+    const riotWebFile = 'src/i18n/strings/' + lang + '.json';
+
+    let translations = {};
+    [reactSdkFile, riotWebFile].forEach(function(f) {
+        if (fs.existsSync(f)) {
+            try {
+                Object.assign(
+                    translations,
+                    JSON.parse(fs.readFileSync(f).toString())
+                );
+            } catch (e) {
+                console.error("Failed: " + f, e);
+                throw e;
+            }
+        }
+    });
+
+    translations = weblateToCounterpart(translations);
+
+    const json = JSON.stringify(translations, null, 4);
+    const jsonBuffer = Buffer.from(json);
+    const digest = loaderUtils.getHashDigest(jsonBuffer, null, null, 7);
+    const filename = `${lang}.${digest}.json`;
+
+    fs.writeFileSync(dest + filename, json);
+    if (verbose) {
+        console.log("Generated language file: " + filename);
+    }
+
+    return filename;
+}
+
+function genLangList(langFileMap) {
+    const languages = {};
+    INCLUDE_LANGS.forEach(function(lang) {
+        const normalizedLanguage = lang.value.toLowerCase().replace("_", "-");
+        const languageParts = normalizedLanguage.split('-');
+        if (languageParts.length == 2 && languageParts[0] == languageParts[1]) {
+            languages[languageParts[0]] = {'fileName': langFileMap[lang.value], 'label': lang.label};
+        } else {
+            languages[normalizedLanguage] = {'fileName': langFileMap[lang.value], 'label': lang.label};
+        }
+    });
+    fs.writeFile('webapp/i18n/languages.json', JSON.stringify(languages, null, 4), function(err) {
+        if (err) {
+            console.error("Copy Error occured: " + err);
+            throw new Error("Failed to generate languages.json");
+        }
+    });
+    if (verbose) {
+        console.log("Generated languages.json");
+    }
+}
+
+/**
+ * Convert translation key from weblate format
+ * (which only supports a single level) to counterpart
+ * which requires object values for 'count' translations.
+ *
+ * eg.
+ *     "there are %(count)s badgers|one": "a badger",
+ *     "there are %(count)s badgers|other": "%(count)s badgers"
+ *   becomes
+ *     "there are %(count)s badgers": {
+ *         "one": "a badger",
+ *         "other": "%(count)s badgers"
+ *     }
+ */
+function weblateToCounterpart(inTrs) {
+    const outTrs = {};
+
+    for (const key of Object.keys(inTrs)) {
+        const keyParts = key.split('|', 2);
+        if (keyParts.length === 2) {
+            let obj = outTrs[keyParts[0]];
+            if (obj === undefined) {
+                obj = {};
+                outTrs[keyParts[0]] = obj;
+            }
+            obj[keyParts[1]] = inTrs[key];
+        } else {
+            outTrs[key] = inTrs[key];
+        }
+    }
+
+    return outTrs;
+}
+
+/**
+watch the input files for a given language,
+regenerate the file, adding its content-hashed filename to langFileMap
+and regenerating languages.json with the new filename
+*/
+function watchLanguage(lang, dest, langFileMap) {
+    const reactSdkFile = 'node_modules/matrix-react-sdk/src/i18n/strings/' + lang + '.json';
+    const riotWebFile = 'src/i18n/strings/' + lang + '.json';
+
+    // XXX: Use a debounce because for some reason if we read the language
+    // file immediately after the FS event is received, the file contents
+    // appears empty. Possibly https://github.com/nodejs/node/issues/6112
+    let makeLangDebouncer;
+    const makeLang = () => {
+        if (makeLangDebouncer) {
+            clearTimeout(makeLangDebouncer);
+        }
+        makeLangDebouncer = setTimeout(() => {
+            const filename = genLangFile(lang, dest);
+            langFileMap[lang]=filename;
+            genLangList(langFileMap);
+        }, 500);
+    };
+
+    [reactSdkFile, riotWebFile].forEach(function(f) {
+        chokidar.watch(f)
+            .on('add', makeLang)
+            .on('change', makeLang)
+            .on('error', errCheck);
+    });
+}
+
+// language resources
+const I18N_DEST = "webapp/i18n/";
+const I18N_FILENAME_MAP = INCLUDE_LANGS.reduce((m, l) => {
+    const filename = genLangFile(l.value, I18N_DEST);
+    m[l.value] = filename;
+    return m;
+}, {});
+genLangList(I18N_FILENAME_MAP);
+
+if (watch) {
+    INCLUDE_LANGS.forEach(l => watchLanguage(l.value, I18N_DEST, I18N_FILENAME_MAP));
+}
+
+// non-language resources
+next(0);

scripts/deploy.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+#
+# download and unpack a riot-web tarball.
+#
+# Allows `bundles` to be extracted to a common directory, and a link to
+# config.json to be added.
+
+from __future__ import print_function
+
+import argparse
+import os
+import os.path
+import subprocess
+import sys
+import tarfile
+import shutil
+import glob
+
+try:
+    # python3
+    from urllib.request import urlretrieve
+except ImportError:
+    # python2
+    from urllib import urlretrieve
+
+class DeployException(Exception):
+    pass
+
+def create_relative_symlink(linkname, target):
+    relpath = os.path.relpath(target, os.path.dirname(linkname))
+    print ("Symlink %s -> %s" % (linkname, relpath))
+    os.symlink(relpath, linkname)
+
+
+def move_bundles(source, dest):
+    """Move the contents of the 'bundles' directory to a common dir
+
+    We check that we will not be overwriting anything before we proceed.
+
+    Args:
+        source (str): path to 'bundles' within the extracted tarball
+        dest (str): target common directory
+    """
+
+    if not os.path.isdir(dest):
+        os.mkdir(dest)
+
+    # build a map from source to destination, checking for non-existence as we go.
+    renames = {}
+    for f in os.listdir(source):
+        dst = os.path.join(dest, f)
+        if os.path.exists(dst):
+            print (
+                "Skipping bundle. The bundle includes '%s' which we have previously deployed."
+                % f
+            )
+        else:
+            renames[os.path.join(source, f)] = dst
+
+    for (src, dst) in renames.iteritems():
+        print ("Move %s -> %s" % (src, dst))
+        os.rename(src, dst)
+
+class Deployer:
+    def __init__(self):
+        self.packages_path = "."
+        self.bundles_path = None
+        self.should_clean = False
+        # filename -> symlink path e.g 'config.localhost.json' => '../localhost/config.json'
+        self.symlink_paths = {}
+        self.verify_signature = True
+
+    def deploy(self, tarball, extract_path):
+        """Download a tarball if necessary, and unpack it
+
+        Returns:
+            (str) the path to the unpacked deployment
+        """
+        print("Deploying %s to %s" % (tarball, extract_path))
+
+        name_str = os.path.basename(tarball).replace(".tar.gz", "")
+        extracted_dir = os.path.join(extract_path, name_str)
+        if os.path.exists(extracted_dir):
+            raise DeployException('Cannot unpack %s: %s already exists' % (
+                tarball, extracted_dir))
+
+        downloaded = False
+        if tarball.startswith("http://") or tarball.startswith("https://"):
+            tarball = self.download_and_verify(tarball)
+            print("Downloaded file: %s" % tarball)
+            downloaded = True
+
+        try:
+            with tarfile.open(tarball) as tar:
+                tar.extractall(extract_path)
+        finally:
+            if self.should_clean and downloaded:
+                os.remove(tarball)
+
+        print ("Extracted into: %s" % extracted_dir)
+
+        if self.symlink_paths:
+            for link_path, file_path in self.symlink_paths.iteritems():
+                create_relative_symlink(
+                    target=file_path,
+                    linkname=os.path.join(extracted_dir, link_path)
+                )
+
+        if self.bundles_path:
+            extracted_bundles = os.path.join(extracted_dir, 'bundles')
+            move_bundles(source=extracted_bundles, dest=self.bundles_path)
+
+            # replace the extracted_bundles dir (which may not be empty if some
+            # bundles were skipped) with a symlink to the common dir.
+            shutil.rmtree(extracted_bundles)
+            create_relative_symlink(
+                target=self.bundles_path,
+                linkname=extracted_bundles,
+            )
+        return extracted_dir
+
+    def download_and_verify(self, url):
+        tarball = self.download_file(url)
+
+        if self.verify_signature:
+            sigfile = self.download_file(url + ".asc")
+            subprocess.check_call(["gpg", "--verify", sigfile, tarball])
+
+        return tarball
+
+    def download_file(self, url):
+        if not os.path.isdir(self.packages_path):
+            os.mkdir(self.packages_path)
+        local_filename = os.path.join(self.packages_path,
+                                      url.split('/')[-1])
+        sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
+        sys.stdout.flush()
+        urlretrieve(url, local_filename)
+        print ("Done")
+        return local_filename
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
+    parser.add_argument(
+        "-p", "--packages-dir", default="./packages", help=(
+            "The directory to download the tarball into. (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "-e", "--extract-path", default="./deploys", help=(
+            "The location to extract .tar.gz files to. (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "-b", "--bundles-dir", nargs='?', default="./bundles", help=(
+            "A directory to move the contents of the 'bundles' directory to. A \
+            symlink to the bundles directory will also be written inside the \
+            extracted tarball. Example: './bundles'. \
+            (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "-c", "--clean", action="store_true", default=False, help=(
+            "Remove .tar.gz files after they have been downloaded and extracted. \
+            (Default: %(default)s)"
+        )
+    )
+    parser.add_argument(
+        "--include", nargs='*', default=['./config*.json'], help=(
+            "Symlink these files into the root of the deployed tarball. \
+             Useful for config files and home pages. Supports glob syntax. \
+             (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "tarball", help=(
+            "filename of tarball, or URL to download."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    deployer = Deployer()
+    deployer.packages_path = args.packages_dir
+    deployer.bundles_path = args.bundles_dir
+    deployer.should_clean = args.clean
+
+    for include in args.include:
+        deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })
+
+    deployer.deploy(args.tarball, args.extract_path)

scripts/docker-link-repos.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -ex
+
+# Automatically link to develop if we're building develop, but only if the caller
+# hasn't asked us to build something else
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+if [ $USE_CUSTOM_SDKS == false ] && [ $BRANCH == 'develop' ]
+then
+    echo "using develop dependencies for react-sdk and js-sdk"
+    USE_CUSTOM_SDKS=true
+    JS_SDK_BRANCH='develop'
+    REACT_SDK_BRANCH='develop'
+fi
+
+if [ $USE_CUSTOM_SDKS == false ]
+then
+    echo "skipping react-sdk and js-sdk installs: USE_CUSTOM_SDKS is false"
+    exit 0
+fi
+
+echo "Linking js-sdk"
+git clone --depth 1 --branch $JS_SDK_BRANCH $JS_SDK_REPO js-sdk
+cd js-sdk
+yarn link
+yarn --network-timeout=100000 install
+cd ../
+
+echo "Linking react-sdk"
+git clone --depth 1 --branch $REACT_SDK_BRANCH $REACT_SDK_REPO react-sdk
+cd react-sdk
+yarn link
+yarn link matrix-js-sdk
+yarn --network-timeout=100000 install
+cd ../
+
+echo "Setting up element-web with react-sdk and js-sdk packages"
+yarn link matrix-js-sdk
+yarn link matrix-react-sdk

scripts/docker-write-version.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -ex
+
+TAG=$(git describe --tags)
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+DIST_VERSION=$TAG
+
+# If the branch comes out as HEAD then we're probably checked out to a tag, so if the thing is *not*
+# coming out as HEAD then we're on a branch. When we're on a branch, we want to resolve ourselves to
+# a few SHAs rather than a version.
+# Docker Hub doesn't always check out the tag and sometimes checks out the branch, so we should look
+# for an appropriately tagged branch as well (heads/v1.2.3).
+if [[ $BRANCH != HEAD && ! $BRANCH =~ heads/v.+ ]]
+then
+    REACT_SHA=$(cd node_modules/matrix-react-sdk; git rev-parse --short=12 HEAD)
+    JSSDK_SHA=$(cd node_modules/matrix-js-sdk; git rev-parse --short=12 HEAD)
+    VECTOR_SHA=$(git rev-parse --short=12 HEAD) # use the ACTUAL SHA rather than assume develop
+    DIST_VERSION=$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA
+fi
+
+echo $DIST_VERSION > /src/webapp/version

scripts/fetch-develop.deps.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# Fetches the js-sdk and matrix-react-sdk dependencies for development
+# or testing purposes
+# If there exists a branch of that dependency with the same name as
+# the branch the current checkout is on, use that branch. Otherwise,
+# use develop.
+
+set -ex
+
+GIT_CLONE_ARGS=("$@")
+[ -z "$defbranch" ] && defbranch="develop"
+
+# clone a specific branch of a github repo
+function clone() {
+    org=$1
+    repo=$2
+    branch=$3
+
+    # Chop 'origin' off the start as jenkins ends up using
+    # branches on the origin, but this doesn't work if we
+    # specify the branch when cloning.
+    branch=${branch#origin/}
+
+    if [ -n "$branch" ]
+    then
+        echo "Trying to use $org/$repo#$branch"
+        git clone git://github.com/$org/$repo.git $repo --branch $branch \
+            "${GIT_CLONE_ARGS[@]}"
+        return $?
+    fi
+    return 1
+}
+
+function dodep() {
+    deforg=$1
+    defrepo=$2
+    rm -rf $defrepo
+
+    # Try the PR author's branch in case it exists on the deps as well.
+    # Try the target branch of the push or PR.
+    # Use the default branch as the last resort.
+    if [[ "$BUILDKITE" == true ]]; then
+        # If BUILDKITE_BRANCH is set, it will contain either:
+        #   * "branch" when the author's branch and target branch are in the same repo
+        #   * "author:branch" when the author's branch is in their fork
+        # We can split on `:` into an array to check.
+        BUILDKITE_BRANCH_ARRAY=(${BUILDKITE_BRANCH//:/ })
+        if [[ "${#BUILDKITE_BRANCH_ARRAY[@]}" == "2" ]]; then
+            prAuthor=${BUILDKITE_BRANCH_ARRAY[0]}
+            prBranch=${BUILDKITE_BRANCH_ARRAY[1]}
+        else
+            prAuthor=$deforg
+            prBranch=$BUILDKITE_BRANCH
+        fi
+        clone $prAuthor $defrepo $prBranch ||
+        clone $deforg $defrepo $BUILDKITE_PULL_REQUEST_BASE_BRANCH ||
+        clone $deforg $defrepo $defbranch ||
+        return $?
+    else
+        clone $deforg $defrepo $ghprbSourceBranch ||
+        clone $deforg $defrepo $GIT_BRANCH ||
+        clone $deforg $defrepo `git rev-parse --abbrev-ref HEAD` ||
+        clone $deforg $defrepo $defbranch ||
+        return $?
+    fi
+
+    echo "$defrepo set to branch "`git -C "$defrepo" rev-parse --abbrev-ref HEAD`
+}
+
+##############################
+
+echo 'Setting up matrix-js-sdk'
+
+dodep matrix-org matrix-js-sdk
+
+pushd matrix-js-sdk
+yarn link
+yarn install
+yarn build
+popd
+
+yarn link matrix-js-sdk
+
+##############################
+
+echo 'Setting up matrix-react-sdk'
+
+dodep matrix-org matrix-react-sdk
+
+pushd matrix-react-sdk
+yarn link
+yarn link matrix-js-sdk
+yarn install
+yarn build
+popd
+
+yarn link matrix-react-sdk
+
+##############################
+
+# Link the reskindex binary in place: if we used `yarn link`,
+# Yarn would do this for us, but we don't because we'd have
+# to define the Yarn binary prefix somewhere so it could put the
+# intermediate symlinks there. Instead, we do it ourselves.
+mkdir -p node_modules/.bin
+ln -sfv ../matrix-react-sdk/scripts/reskindex.js node_modules/.bin/reskindex

scripts/genflags.sh

@@ -0,0 +1,70 @@
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# genflags.sh - Generates pngs for use with CountryDropdown.js
+#
+# Dependencies:
+#   - imagemagick --with-rsvg (because default imagemagick SVG
+#       renderer does not produce accurate results)
+#
+# on macOS, this is most easily done with:
+#   brew install imagemagick --with-librsvg
+#
+# This will clone the googlei18n flag repo before converting
+# all phonenumber.js-supported country flags (as SVGs) into
+# PNGs that can be used by CountryDropdown.js.
+
+# Allow CTRL+C to terminate the script
+trap "echo Exited!; exit;" SIGINT SIGTERM
+
+# git clone the google repo to get flag SVGs
+git clone git@github.com:googlei18n/region-flags
+for f in region-flags/svg/*.svg; do
+    # Skip state flags
+    if [[ $f =~ [A-Z]{2}-[A-Z]{2,3}.svg ]] ; then
+        echo "Skipping state flag "$f
+        continue
+    fi
+
+    # Skip countries not included in phonenumber.js
+    if [[ $f =~ (AC|CP|DG|EA|EU|IC|TA|UM|UN|XK).svg ]] ; then
+        echo "Skipping non-phonenumber supported flag "$f
+        continue
+    fi
+
+    # Run imagemagick convert
+    #   -background none : transparent background
+    #   -resize 50x30    : resize the flag to have a height of 15px (2x)
+    #       By default, aspect ratio is respected so the width will
+    #       be correct and not necessarily 25px.
+    #   -filter Lanczos  : use sharper resampling to avoid muddiness
+    #   -gravity Center  : keep the image central when adding an -extent
+    #   -border 1        : add a 1px border around the flag
+    #   -bordercolor     : set the border colour
+    #   -extent 54x54    : surround the image with padding so that it
+    #       has the dimensions 27x27px (2x).
+    convert $f -background none -filter Lanczos -resize 50x30 \
+        -gravity Center -border 1 -bordercolor \#e0e0e0 \
+        -extent 54x54 $f.png
+
+    # $f.png will be region-flags/svg/XX.svg.png at this point
+
+    # Extract filename from path $f
+    newname=${f##*/}
+    # Replace .svg with .png
+    newname=${newname%.svg}.png
+    # Move the file to flags directory
+    mv $f.png ../res/flags/$newname
+    echo "Generated res/flags/"$newname
+done

scripts/issues-burndown.pl

@@ -0,0 +1,136 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use Net::GitHub;
+use Time::Moment;
+use Term::ReadPassword;
+
+# This version of the script emits the cumulative number of bugs, split into open & closed
+# suitable for drawing the 'top' and 'bottom' of a burndown graph.
+#
+# N.B. this doesn't take into account issues changing priority over time, but only their most recent priority.
+#
+# If you want instead the number of open issues on a given day, then look at issues-no-state.pl
+
+my $gh = Net::GitHub->new(
+    login => 'ara4n', pass => read_password("github password: "),
+);
+
+$gh->set_default_user_repo('vector-im', 'element-web');
+
+#my @issues = $gh->issue->repos_issues({ state => 'all', milestone => 3 });
+my @issues = $gh->issue->repos_issues({ state => 'all' });
+while ($gh->issue->has_next_page) {
+    push @issues, $gh->issue->next_page;
+}
+
+# we want:
+# day by day:
+# split by { open, closed }
+# split by { bug, feature, neither }
+# each split by { p1, p2, p3, p4, p5, unprioritised }  <- priority
+# each split by { minor, major, critical, cosmetic, network, no-severity }  <- severity
+# then split (with overlap between the groups) as { total, tag1, tag2, ... }?
+
+# ...and then all over again split by milestone.
+
+my $days = {};
+my $schema = {};
+my $now = Time::Moment->now;
+
+foreach my $issue (@issues) {
+    next if ($issue->{pull_request});
+
+    # use Data::Dumper;
+    # print STDERR Dumper($issue);
+
+    my @label_list = map { $_->{name} } @{$issue->{labels}};
+    my $labels = {};
+    $labels->{$_} = 1 foreach (@label_list);
+    $labels->{bug}++ if ($labels->{cosmetic} && !$labels->{bug} && !$labels->{feature});
+
+    my $extract_labels = sub {
+        my $label = undef;
+        foreach (@_) {
+            $label ||= $_ if (delete $labels->{$_});
+        }
+        return $label;
+    };
+
+    my $state = $issue->{state};
+    my $type = &$extract_labels(qw(bug feature)) || "neither";
+    my $priority = &$extract_labels(qw(p1 p2 p3 p4 p5)) || "unprioritised";
+    my $severity = &$extract_labels(qw(minor major critical cosmetic network)) || "no-severity";
+
+    my $start = Time::Moment->from_string($issue->{created_at});
+
+    do {
+        my $ymd = $start->strftime('%F');
+
+        $days->{ $ymd }->{ 'created' }->{ $type }->{ $priority }->{ $severity }->{ total }++;
+        $schema->{ 'created' }->{ $type }->{ $priority }->{ $severity }->{ total }++;
+        foreach (keys %$labels) {
+            $days->{ $ymd }->{ 'created' }->{ $type }->{ $priority }->{ $severity }->{ $_ }++;
+            $schema->{ 'created' }->{ $type }->{ $priority }->{ $severity }->{ $_ }++;
+        }
+
+        $start = $start->plus_days(1);
+        # print STDERR "^";
+    } while ($start->compare($now) < 0);
+
+    if ($state eq 'closed') {
+        my $end = Time::Moment->from_string($issue->{closed_at});
+        do {
+            my $ymd = $end->strftime('%F');
+
+            $days->{ $ymd }->{ 'resolved' }->{ $type }->{ $priority }->{ $severity }->{ total }++;
+            $schema->{ 'resolved' }->{ $type }->{ $priority }->{ $severity }->{ total }++;
+            foreach (keys %$labels) {
+                $days->{ $ymd }->{ 'resolved' }->{ $type }->{ $priority }->{ $severity }->{ $_ }++;
+                $schema->{ 'resolved' }->{ $type }->{ $priority }->{ $severity }->{ $_ }++;
+            }
+
+            $end = $end->plus_days(1);
+        } while ($end->compare($now) < 0);
+        # print STDERR "v";
+    }
+
+    # print STDERR "\n";
+}
+
+print "day,";
+foreach my $state (sort keys %{$schema}) {
+    foreach my $type (grep { /^(bug|feature)$/ } sort keys %{$schema->{$state}}) {
+        foreach my $priority (grep { /^(p1|p2)$/ } sort keys %{$schema->{$state}->{$type}}) {
+            foreach my $severity (sort keys %{$schema->{$state}->{$type}->{$priority}}) {
+                # foreach my $tag (sort keys %{$schema->{$state}->{$type}->{$priority}->{$severity}}) {
+                #     print "\"$type\n$priority\n$severity\n$tag\",";
+                # }
+                print "\"$state\n$type\n$priority\n$severity\",";
+            }
+        }
+    }
+}
+print "\n";
+
+foreach my $day (sort keys %$days) {
+    print "$day,";
+    foreach my $state (sort keys %{$schema}) {
+        foreach my $type (grep { /^(bug|feature)$/ } sort keys %{$schema->{$state}}) {
+            foreach my $priority (grep { /^(p1|p2)$/ } sort keys %{$schema->{$state}->{$type}}) {
+                foreach my $severity (sort keys %{$schema->{$state}->{$type}->{$priority}}) {
+                    # foreach my $tag (sort keys %{$schema->{$state}->{$type}->{$priority}->{$severity}}) {
+                    #     print $days->{$day}->{$state}->{$type}->{$priority}->{$severity}->{$tag} || 0;
+                    #     print ",";
+                    # }
+                    print $days->{$day}->{$state}->{$type}->{$priority}->{$severity}->{total} || 0;
+                    print ",";
+                }
+            }
+        }
+    }
+    print "\n";
+}
+

scripts/issues-no-state.pl

@@ -0,0 +1,112 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use Net::GitHub;
+use DateTime;
+use DateTime::Format::ISO8601;
+use Term::ReadPassword;
+
+# This version of the script emits the total number of bugs open on a given day,
+# split by various tags.
+#
+# If you want instead the cumulative number of open & closed issues on a given day,
+# then look at issues-burndown.pl
+
+my $gh = Net::GitHub->new(
+    login => 'ara4n', pass => read_password("github password: "),
+);
+
+$gh->set_default_user_repo('vector-im', 'riot-web');
+
+#my @issues = $gh->issue->repos_issues({ state => 'all', milestone => 3 });
+my @issues = $gh->issue->repos_issues({ state => 'all' });
+while ($gh->issue->has_next_page) {
+    push @issues, $gh->issue->next_page;
+}
+
+# we want:
+# day by day:
+# split by { open, closed }
+# split by { bug, feature, neither }
+# each split by { p1, p2, p3, p4, p5, unprioritised }  <- priority
+# each split by { minor, major, critical, cosmetic, network, no-severity }  <- severity
+# then split (with overlap between the groups) as { total, tag1, tag2, ... }?
+
+# ...and then all over again split by milestone.
+
+my $days = {};
+my $schema = {};
+my $now = DateTime->now();
+
+foreach my $issue (@issues) {
+    next if ($issue->{pull_request});
+
+    use Data::Dumper;
+    print STDERR Dumper($issue);
+
+    my @label_list = map { $_->{name} } @{$issue->{labels}};
+    my $labels = {};
+    $labels->{$_} = 1 foreach (@label_list);
+    $labels->{bug}++ if ($labels->{cosmetic} && !$labels->{bug} && !$labels->{feature});
+
+    my $extract_labels = sub {
+        my $label = undef;
+        foreach (@_) {
+            $label ||= $_ if (delete $labels->{$_});
+        }
+        return $label;
+    };
+
+    my $type = &$extract_labels(qw(bug feature)) || "neither";
+    my $priority = &$extract_labels(qw(p1 p2 p3 p4 p5)) || "unprioritised";
+    my $severity = &$extract_labels(qw(minor major critical cosmetic network)) || "no-severity";
+
+    my $start = DateTime::Format::ISO8601->parse_datetime($issue->{created_at});
+    my $end = $issue->{closed_at} ? DateTime::Format::ISO8601->parse_datetime($issue->{closed_at}) : $now;
+
+    do {
+        my $ymd = $start->ymd();
+
+        $days->{ $ymd }->{ $type }->{ $priority }->{ $severity }->{ total }++;
+        $schema->{ $type }->{ $priority }->{ $severity }->{ total }++;
+        foreach (keys %$labels) {
+            $days->{ $ymd }->{ $type }->{ $priority }->{ $severity }->{ $_ }++;
+            $schema->{ $type }->{ $priority }->{ $severity }->{ $_ }++;
+        }
+
+        $start = $start->add(days => 1);
+    } while (DateTime->compare($start, $end) < 0);
+}
+
+print "day,";
+foreach my $type (sort keys %{$schema}) {
+    foreach my $priority (sort keys %{$schema->{$type}}) {
+        foreach my $severity (sort keys %{$schema->{$type}->{$priority}}) {
+            # foreach my $tag (sort keys %{$schema->{$type}->{$priority}->{$severity}}) {
+            #     print "\"$type\n$priority\n$severity\n$tag\",";
+            # }
+            print "\"$type\n$priority\n$severity\",";
+        }
+    }
+}
+print "\n";
+
+foreach my $day (sort keys %$days) {
+    print "$day,";
+    foreach my $type (sort keys %{$schema}) {
+        foreach my $priority (sort keys %{$schema->{$type}}) {
+            foreach my $severity (sort keys %{$schema->{$type}->{$priority}}) {
+                # foreach my $tag (sort keys %{$schema->{$type}->{$priority}->{$severity}}) {
+                #     print $days->{$day}->{$type}->{$priority}->{$severity}->{$tag} || 0;
+                #     print ",";
+                # }
+                print $days->{$day}->{$type}->{$priority}->{$severity}->{total} || 0;
+                print ",";
+            }
+        }
+    }
+    print "\n";
+}
+

scripts/make-icons.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+#
+# Converts an svg logo into the various image resources required by
+# the various platforms deployments.
+#
+# On debian-based systems you need these deps:
+#   apt-get install xmlstarlet python3-cairosvg icnsutils
+
+if [ $# != 1 ]
+then
+    echo "Usage: $0 <svg file>"
+    exit
+fi
+
+set -e
+set -x
+
+tmpdir=`mktemp -d 2>/dev/null || mktemp -d -t 'icontmp'`
+
+for i in 1024 512 310 256 192 180 152 150 144 128 120 114 96 76 72 70 64 60 57 48 36 32 24 16
+do
+    #convert -background none -density 1000 -resize $i -extent $i -gravity center "$1" "$tmpdir/$i.png"
+
+    # Above is the imagemagick command to render an svg to png. Unfortunately, its support for SVGs
+    # with CSS isn't very good (with rsvg and even moreso the built in renderer) so we use cairosvg.
+    # This can be installed with:
+    #    pip install cairosvg==1.0.22 # Version 2 doesn't support python 2
+    #    pip install tinycss
+    #    pip install cssselect # These are necessary for CSS support
+    # You'll also need xmlstarlet from your favourite package manager
+    #
+    # Cairosvg doesn't suport rendering at a specific size (https://github.com/Kozea/CairoSVG/issues/83#issuecomment-215720176)
+    # so we have to 'resize the svg' first (add width and height attributes to the svg element) to make it render at the
+    # size we need.
+    # XXX: This will break if the svg already has width and height attributes
+    cp "$1" "$tmpdir/tmp.svg"
+    xmlstarlet ed -N x="http://www.w3.org/2000/svg" --insert "/x:svg" --type attr -n width -v $i "$tmpdir/tmp.svg" > "$tmpdir/tmp2.svg"
+    xmlstarlet ed -N x="http://www.w3.org/2000/svg" --insert "/x:svg" --type attr -n height -v $i "$tmpdir/tmp2.svg" > "$tmpdir/tmp3.svg"
+    cairosvg -f png -o "$tmpdir/$i.png"  "$tmpdir/tmp3.svg"
+    rm "$tmpdir/tmp.svg" "$tmpdir/tmp2.svg" "$tmpdir/tmp3.svg"
+done
+
+# one more for the non-square mstile
+cp "$1" "$tmpdir/tmp.svg"
+xmlstarlet ed -N x="http://www.w3.org/2000/svg" --insert "/x:svg" --type attr -n width -v 310 "$tmpdir/tmp.svg" > "$tmpdir/tmp2.svg"
+xmlstarlet ed -N x="http://www.w3.org/2000/svg" --insert "/x:svg" --type attr -n height -v 150 "$tmpdir/tmp2.svg" > "$tmpdir/tmp3.svg"
+cairosvg -f png -o "$tmpdir/310x150.png"  "$tmpdir/tmp3.svg"
+rm "$tmpdir/tmp.svg" "$tmpdir/tmp2.svg" "$tmpdir/tmp3.svg"
+
+mkdir "$tmpdir/Riot.iconset"
+cp "$tmpdir/16.png" "$tmpdir/Riot.iconset/icon_16x16.png"
+cp "$tmpdir/32.png" "$tmpdir/Riot.iconset/icon_16x16@2x.png"
+cp "$tmpdir/32.png" "$tmpdir/Riot.iconset/icon_32x32.png"
+cp "$tmpdir/64.png" "$tmpdir/Riot.iconset/icon_32x32@2x.png"
+cp "$tmpdir/128.png" "$tmpdir/Riot.iconset/icon_128x128.png"
+cp "$tmpdir/256.png" "$tmpdir/Riot.iconset/icon_128x128@2x.png"
+cp "$tmpdir/256.png" "$tmpdir/Riot.iconset/icon_256x256.png"
+cp "$tmpdir/512.png" "$tmpdir/Riot.iconset/icon_256x256@2x.png"
+cp "$tmpdir/512.png" "$tmpdir/Riot.iconset/icon_512x512.png"
+cp "$tmpdir/1024.png" "$tmpdir/Riot.iconset/icon_512x512@2x.png"
+
+if [ -x "$(command -v iconutil)" ]; then
+  # available on macos
+  iconutil -c icns -o electron_app/build/icon.icns "$tmpdir/Riot.iconset"
+elif [ -x "$(command -v png2icns)" ]; then
+  # available on linux
+  # png2icns is more finicky about its input than iconutil
+  # 1. it doesn't support a 64x64 (aka 32x32@2x)
+  # 2. it doesn't like duplicates (128x128@2x == 256x256)
+  rm "$tmpdir/Riot.iconset/icon_128x128@2x.png"
+  rm "$tmpdir/Riot.iconset/icon_256x256@2x.png"
+  rm "$tmpdir/Riot.iconset/icon_16x16@2x.png"
+  rm "$tmpdir/Riot.iconset/icon_32x32@2x.png"
+  png2icns electron_app/build/icon.icns "$tmpdir"/Riot.iconset/*png
+else
+  echo "WARNING: Unsupported platform. Skipping icns build"
+fi
+
+cp "$tmpdir/36.png" "res/vector-icons/android-chrome-36x36.png"
+cp "$tmpdir/48.png" "res/vector-icons/android-chrome-48x48.png"
+cp "$tmpdir/72.png" "res/vector-icons/android-chrome-72x72.png"
+cp "$tmpdir/96.png" "res/vector-icons/android-chrome-96x96.png"
+cp "$tmpdir/144.png" "res/vector-icons/android-chrome-144x144.png"
+cp "$tmpdir/192.png" "res/vector-icons/android-chrome-192x192.png"
+cp "$tmpdir/180.png" "res/vector-icons/apple-touch-icon.png"
+cp "$tmpdir/180.png" "res/vector-icons/apple-touch-icon-precomposed.png"
+cp "$tmpdir/57.png" "res/vector-icons/apple-touch-icon-57x57.png"
+cp "$tmpdir/60.png" "res/vector-icons/apple-touch-icon-60x60.png"
+cp "$tmpdir/72.png" "res/vector-icons/apple-touch-icon-72x72.png"
+cp "$tmpdir/76.png" "res/vector-icons/apple-touch-icon-76x76.png"
+cp "$tmpdir/114.png" "res/vector-icons/apple-touch-icon-114x114.png"
+cp "$tmpdir/120.png" "res/vector-icons/apple-touch-icon-120x120.png"
+cp "$tmpdir/144.png" "res/vector-icons/apple-touch-icon-144x144.png"
+cp "$tmpdir/152.png" "res/vector-icons/apple-touch-icon-152x152.png"
+cp "$tmpdir/180.png" "res/vector-icons/apple-touch-icon-180x180.png"
+cp "$tmpdir/16.png" "res/vector-icons/favicon-16x16.png"
+cp "$tmpdir/32.png" "res/vector-icons/favicon-32x32.png"
+cp "$tmpdir/96.png" "res/vector-icons/favicon-96x96.png"
+cp "$tmpdir/70.png" "res/vector-icons/mstile-70x70.png"
+cp "$tmpdir/144.png" "res/vector-icons/mstile-144x144.png"
+cp "$tmpdir/150.png" "res/vector-icons/mstile-150x150.png"
+cp "$tmpdir/310.png" "res/vector-icons/mstile-310x310.png"
+cp "$tmpdir/310x150.png" "res/vector-icons/mstile-310x150.png"
+cp "$tmpdir/180.png" "electron_app/img/riot.png"
+
+convert "$tmpdir/16.png" "$tmpdir/32.png" "$tmpdir/64.png" "$tmpdir/128.png"  "$tmpdir/256.png" "res/vector-icons/favicon.ico"
+
+cp "res/vector-icons/favicon.ico" "electron_app/build/icon.ico"
+cp "res/vector-icons/favicon.ico" "electron_app/img/riot.ico"
+
+# https://github.com/electron-userland/electron-builder/blob/3f97b86993d4ea5172e562b182230a194de0f621/src/targets/LinuxTargetHelper.ts#L127
+for i in 24 96 16 48 64 128 256 512
+do
+    cp "$tmpdir/$i.png" "electron_app/build/icons/${i}x${i}.png"
+done
+
+rm -r "$tmpdir"

scripts/package.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+if [ -n "$DIST_VERSION" ]; then
+    version=$DIST_VERSION
+else
+    version=`git describe --dirty --tags || echo unknown`
+fi
+
+yarn clean
+yarn build
+
+# include the sample config in the tarball. Arguably this should be done by
+# `yarn build`, but it's just too painful.
+cp config.sample.json webapp/
+
+mkdir -p dist
+cp -r webapp riot-$version
+
+# Just in case you have a local config, remove it before packaging
+rm riot-$version/config.json || true
+
+# if $version looks like semver with leading v, strip it before writing to file
+if [[ ${version} =~ ^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-.+)?$ ]]; then
+    echo ${version:1} > riot-$version/version
+else
+    echo ${version} > riot-$version/version
+fi
+
+tar chvzf dist/riot-$version.tar.gz riot-$version
+rm -r riot-$version
+
+echo
+echo "Packaged dist/riot-$version.tar.gz"

scripts/redeploy.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python
+#
+# auto-deploy script for https://develop.element.io
+#
+# Listens for buildkite webhook pokes (https://buildkite.com/docs/apis/webhooks)
+# When it gets one, downloads the artifact from buildkite
+# and deploys it as the new version.
+#
+# Requires the following python packages:
+#
+#   - requests
+#   - flask
+#
+from __future__ import print_function
+import json, requests, tarfile, argparse, os, errno
+import time
+import traceback
+from urlparse import urljoin
+import glob
+import re
+import shutil
+import threading
+from Queue import Queue
+
+from flask import Flask, jsonify, request, abort
+
+from deploy import Deployer, DeployException
+
+app = Flask(__name__)
+
+deployer = None
+arg_extract_path = None
+arg_symlink = None
+arg_webhook_token = None
+arg_api_token = None
+
+workQueue = Queue()
+
+def create_symlink(source, linkname):
+    try:
+        os.symlink(source, linkname)
+    except OSError, e:
+        if e.errno == errno.EEXIST:
+            # atomic modification
+            os.symlink(source, linkname + ".tmp")
+            os.rename(linkname + ".tmp", linkname)
+        else:
+            raise e
+
+def req_headers():
+    return {
+        "Authorization": "Bearer %s" % (arg_api_token,),
+    }
+
+# Buildkite considers a poke to have failed if it has to wait more than 10s for
+# data (any data, not just the initial response) and it normally takes longer than
+# that to download an artifact from buildkite. Apparently there is no way in flask
+# to finish the response and then keep doing stuff, so instead this has to involve
+# threading. Sigh.
+def worker_thread():
+    while True:
+        toDeploy = workQueue.get()
+        deploy_buildkite_artifact(*toDeploy)
+
+@app.route("/", methods=["POST"])
+def on_receive_buildkite_poke():
+    got_webhook_token = request.headers.get('X-Buildkite-Token')
+    if got_webhook_token != arg_webbook_token:
+        print("Denying request with incorrect webhook token: %s" % (got_webhook_token,))
+        abort(400, "Incorrect webhook token")
+        return
+
+    required_api_prefix = None
+    if arg_buildkite_org is not None:
+        required_api_prefix = 'https://api.buildkite.com/v2/organizations/%s' % (arg_buildkite_org,)
+
+    incoming_json = request.get_json()
+    if not incoming_json:
+        abort(400, "No JSON provided!")
+        return
+    print("Incoming JSON: %s" % (incoming_json,))
+
+    event = incoming_json.get("event")
+    if event is None:
+        abort(400, "No 'event' specified")
+        return
+
+    if event == 'ping':
+        print("Got ping request - responding")
+        return jsonify({'response': 'pong!'})
+
+    if event != 'build.finished':
+        print("Rejecting '%s' event")
+        abort(400, "Unrecognised event")
+        return
+
+    build_obj = incoming_json.get("build")
+    if build_obj is None:
+        abort(400, "No 'build' object")
+        return
+
+    build_url = build_obj.get('url')
+    if build_url is None:
+        abort(400, "build has no url")
+        return
+
+    if required_api_prefix is not None and not build_url.startswith(required_api_prefix):
+        print("Denying poke for build url with incorrect prefix: %s" % (build_url,))
+        abort(400, "Invalid build url")
+        return
+
+    build_num = build_obj.get('number')
+    if build_num is None:
+        abort(400, "build has no number")
+        return
+
+    pipeline_obj = incoming_json.get("pipeline")
+    if pipeline_obj is None:
+        abort(400, "No 'pipeline' object")
+        return
+
+    pipeline_name = pipeline_obj.get('name')
+    if pipeline_name is None:
+        abort(400, "pipeline has no name")
+        return
+
+    artifacts_url = build_url + "/artifacts"
+    artifacts_resp = requests.get(artifacts_url, headers=req_headers())
+    artifacts_resp.raise_for_status()
+    artifacts_array = artifacts_resp.json()
+    
+    artifact_to_deploy = None
+    for artifact in artifacts_array:
+        if re.match(r"dist/.*.tar.gz", artifact['path']):
+            artifact_to_deploy = artifact
+    if artifact_to_deploy is None:
+        print("No suitable artifacts found")
+        return jsonify({})
+
+    # double paranoia check: make sure the artifact is on the right org too
+    if required_api_prefix is not None and not artifact_to_deploy['url'].startswith(required_api_prefix):
+        print("Denying poke for build url with incorrect prefix: %s" % (artifact_to_deploy['url'],))
+        abort(400, "Refusing to deploy artifact from URL %s", artifact_to_deploy['url'])
+        return
+
+    # there's no point building up a queue of things to deploy, so if there are any pending jobs,
+    # remove them
+    while not workQueue.empty():
+        try:
+            workQueue.get(False)
+        except:
+            pass
+    workQueue.put([artifact_to_deploy, pipeline_name, build_num])
+
+    return jsonify({})
+
+def deploy_buildkite_artifact(artifact, pipeline_name, build_num):
+    artifact_response = requests.get(artifact['url'], headers=req_headers())
+    artifact_response.raise_for_status()
+    artifact_obj = artifact_response.json()
+
+    # we extract into a directory based on the build number. This avoids the
+    # problem of multiple builds building the same git version and thus having
+    # the same tarball name. That would lead to two potential problems:
+    #   (a) sometimes jenkins serves corrupted artifacts; we would replace
+    #       a good deploy with a bad one
+    #   (b) we'll be overwriting the live deployment, which means people might
+    #       see half-written files.
+    build_dir = os.path.join(arg_extract_path, "%s-#%s" % (pipeline_name, build_num))
+    try:
+        extracted_dir = deploy_tarball(artifact_obj, build_dir)
+    except DeployException as e:
+        traceback.print_exc()
+        abort(400, e.message)
+
+    create_symlink(source=extracted_dir, linkname=arg_symlink)
+
+def deploy_tarball(artifact, build_dir):
+    """Download a tarball from jenkins and unpack it
+
+    Returns:
+        (str) the path to the unpacked deployment
+    """
+    if os.path.exists(build_dir):
+        raise DeployException(
+            "Not deploying. We have previously deployed this build."
+        )
+    os.mkdir(build_dir)
+
+    print("Fetching artifact %s -> %s..." % (artifact['download_url'], artifact['filename']))
+
+    # Download the tarball here as buildkite needs auth to do this
+    # we don't pgp-sign buildkite artifacts, relying on HTTPS and buildkite
+    # not being evil. If that's not good enough for you, don't use develop.element.io.
+    resp = requests.get(artifact['download_url'], stream=True, headers=req_headers())
+    resp.raise_for_status()
+    with open(artifact['filename'], 'wb') as ofp:
+        shutil.copyfileobj(resp.raw, ofp)
+    print("...download complete. Deploying...")
+
+    # we rely on the fact that flask only serves one request at a time to
+    # ensure that we do not overwrite a tarball from a concurrent request.
+
+    return deployer.deploy(artifact['filename'], build_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
+    parser.add_argument(
+        "-p", "--port", dest="port", default=4000, type=int, help=(
+            "The port to listen on for requests from Jenkins."
+        )
+    )
+    parser.add_argument(
+        "-e", "--extract", dest="extract", default="./extracted", help=(
+            "The location to extract .tar.gz files to."
+        )
+    )
+    parser.add_argument(
+        "-b", "--bundles-dir", dest="bundles_dir", help=(
+            "A directory to move the contents of the 'bundles' directory to. A \
+            symlink to the bundles directory will also be written inside the \
+            extracted tarball. Example: './bundles'."
+        )
+    )
+    parser.add_argument(
+        "-c", "--clean", dest="clean", action="store_true", default=False, help=(
+            "Remove .tar.gz files after they have been downloaded and extracted."
+        )
+    )
+    parser.add_argument(
+        "-s", "--symlink", dest="symlink", default="./latest", help=(
+            "Write a symlink to this location pointing to the extracted tarball. \
+            New builds will keep overwriting this symlink. The symlink will point \
+            to the /vector directory INSIDE the tarball."
+        )
+    )
+
+    # --include ../../config.json ./localhost.json homepages/*
+    parser.add_argument(
+        "--include", nargs='*', default='./config*.json', help=(
+            "Symlink these files into the root of the deployed tarball. \
+             Useful for config files and home pages. Supports glob syntax. \
+             (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "--test", dest="tarball_uri", help=(
+            "Don't start an HTTP listener. Instead download a build from Jenkins \
+            immediately."
+        ),
+    )
+
+    parser.add_argument(
+        "--webhook-token", dest="webhook_token", help=(
+            "Only accept pokes with this buildkite token."
+        ), required=True,
+    )
+
+    parser.add_argument(
+        "--api-token", dest="api_token", help=(
+            "API access token for buildkite. Require read_artifacts scope."
+        ), required=True,
+    )
+
+    # We require a matching webhook token, but because we take everything else
+    # about what to deploy from the poke body, we can be a little more paranoid
+    # and only accept builds / artifacts from a specific buildkite org
+    parser.add_argument(
+        "--org", dest="buildkite_org", help=(
+            "Lock down to this buildkite org"
+        )
+    )
+
+    args = parser.parse_args()
+    arg_extract_path = args.extract
+    arg_symlink = args.symlink
+    arg_webbook_token = args.webhook_token
+    arg_api_token = args.api_token
+    arg_buildkite_org = args.buildkite_org
+
+    if not os.path.isdir(arg_extract_path):
+        os.mkdir(arg_extract_path)
+
+    deployer = Deployer()
+    deployer.bundles_path = args.bundles_dir
+    deployer.should_clean = args.clean
+
+    for include in args.include:
+        deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })
+
+    if args.tarball_uri is not None:
+        build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
+        deploy_tarball(args.tarball_uri, build_dir)
+    else:
+        print(
+            "Listening on port %s. Extracting to %s%s. Symlinking to %s. Include files: %s" %
+            (args.port,
+             arg_extract_path,
+             " (clean after)" if deployer.should_clean else "",
+             arg_symlink,
+             deployer.symlink_paths,
+            )
+        )
+        fred = threading.Thread(target=worker_thread)
+        fred.daemon = True
+        fred.start()
+        app.run(port=args.port, debug=False)

scripts/yarn-sub.js

@@ -0,0 +1,22 @@
+const path = require('path');
+const child_process = require('child_process');
+
+const moduleName = process.argv[2];
+if (!moduleName) {
+    console.error("Expected module name");
+    process.exit(1);
+}
+
+const argString = process.argv.length > 3 ? process.argv.slice(3).join(" ") : "";
+if (!argString) {
+    console.error("Expected an yarn argument string to use");
+    process.exit(1);
+}
+
+const modulePath = path.dirname(require.resolve(`${moduleName}/package.json`));
+
+child_process.execSync("yarn " + argString, {
+    env: process.env,
+    cwd: modulePath,
+    stdio: ['inherit', 'inherit', 'inherit'],
+});