Reduce the number of terminals required to build riot-web to 1

A step towards a real solution for https://github.com/vector-im/riot-web/issues/7305

This approach makes use of `npm link` to remove the use of symlinks in the build process. The build process has also been altered to invoke the build process of each underlying SDK (react, js). This means that one can now `npm link` and `npm start` and have a working environment. 

At the same time, parallelshell was dropped due to lack of maintenance from the maintainer.

README.md

@@ -64,12 +64,13 @@ Building From Source
 Riot is a modular webapp built with modern ES6 and requires a npm build system
 to build.
 
-1. Install or update `node.js` so that your `node` is at least v6.3.0 (and `npm`
+1. Install or update `node.js` so that your `node` is at least v8.12.0 (and `npm`
-   is at least v3.10.x).
+   is at least v5.x).
 1. Clone the repo: `git clone https://github.com/vector-im/riot-web.git`.
 1. Switch to the riot-web directory: `cd riot-web`.
-1. If you're using the `develop` branch, install the develop versions of the
+1. If you're using the `develop` branch then it is recommended to set up a proper
-   dependencies, as the released ones will be too old:
+   development environment ("Setting up a dev environment" below) however one can
+   install the develop versions of the dependencies instead:
    ```
    scripts/fetch-develop.deps.sh
    ```
@@ -89,13 +90,9 @@ to build.
    npm install
    npm run build
    ```
-   However, we recommend setting up a proper development environment (see "Setting
+   Or just use https://riot.im/develop - the continuous integration release of the 
-   up a dev environment" below) if you want to run your own copy of the
+   develop branch. (Note that we don't reference the develop versions in git directly
-   `develop` branch, as it makes it much easier to keep these dependencies
+   due to https://github.com/npm/npm/issues/3055.)
-   up-to-date.  Or just use https://riot.im/develop - the continuous integration
-   release of the develop branch.
-   (Note that we don't reference the develop versions in git directly due to
-   https://github.com/npm/npm/issues/3055.)
 1. Install the prerequisites: `npm install`.
 1. Configure the app by copying `config.sample.json` to `config.json` and
    modifying it (see below for details).
@@ -239,7 +236,7 @@ higher and lower level React components useful for building Matrix communication
 apps using React.
 
 After creating a new component you must run `npm run reskindex` to regenerate
-the `component-index.js` for the app (used in future for skinning)
+the `component-index.js` for the app (used in future for skinning).
 
 Please note that Riot is intended to run correctly without access to the public
 internet.  So please don't depend on resources (JS libs, CSS, images, fonts)
@@ -256,20 +253,17 @@ having to manually rebuild each time.
 
 First clone and build `matrix-js-sdk`:
 
-1. `git clone git@github.com:matrix-org/matrix-js-sdk.git`
+1. `git clone https://github.com/matrix-org/matrix-js-sdk.git`
 1. `pushd matrix-js-sdk`
 1. `git checkout develop`
-1. `npm install`
-1. `npm install source-map-loader` # because webpack is made of fail (https://github.com/webpack/webpack/issues/1472)
 1. `popd`
 
 Then similarly with `matrix-react-sdk`:
 
-1. `git clone git@github.com:matrix-org/matrix-react-sdk.git`
+1. `git clone https://github.com/matrix-org/matrix-react-sdk.git`
 1. `pushd matrix-react-sdk`
 1. `git checkout develop`
-1. `npm install`
+1. `npm link ../matrix-js-sdk`
-1. `rm -r node_modules/matrix-js-sdk; ln -s ../../matrix-js-sdk node_modules/`
 1. `popd`
 
 Finally, build and start Riot itself:
@@ -278,8 +272,8 @@ Finally, build and start Riot itself:
 1. `cd riot-web`
 1. `git checkout develop`
 1. `npm install`
-1. `rm -r node_modules/matrix-js-sdk; ln -s ../../matrix-js-sdk node_modules/`
+1. `npm link ../matrix-js-sdk`
-1. `rm -r node_modules/matrix-react-sdk; ln -s ../../matrix-react-sdk node_modules/`
+1. `npm link ../matrix-react-sdk`
 1. `npm start`
 1. Wait a few seconds for the initial build to finish; you should see something like:
     ```
@@ -298,10 +292,8 @@ Finally, build and start Riot itself:
    disables caching, so do NOT use it in production.
 1. Open http://127.0.0.1:8080/ in your browser to see your newly built Riot.
 
-When you make changes to `matrix-react-sdk` or `matrix-js-sdk`, you will need
+When you make changes to `matrix-react-sdk` or `matrix-js-sdk` they should be
-to run `npm run build` in the relevant directory. You can do this automatically
+automatically picked up by webpack and built.
-by instead running `npm start` in the directory, to start a development builder
-which will watch for changes to the files and rebuild automatically.
 
 If you add or remove any components from the Riot skin, you will need to rebuild
 the skin's index by running, `npm run reskindex`.

package.json

@@ -36,16 +36,22 @@
     "build:bundle": "cross-env NODE_ENV=production webpack -p --progress --bail",
     "build:bundle:dev": "webpack --optimize-occurence-order --progress --bail",
     "build:electron": "npm run clean && npm run build && npm run install:electron && build -wml --ia32 --x64",
-    "build": "npm run reskindex && npm run build:res && npm run build:bundle",
+    "build:react-sdk": "node scripts/build-watch-sdk.js build react",
-    "build:dev": "npm run reskindex && npm run build:res && npm run build:bundle:dev",
+    "build:js-sdk": "node scripts//build-watch-sdk.js build js",
+    "build": "npm run build:js-sdk && npm run build:react-sdk && npm run reskindex && npm run build:res && npm run build:bundle",
+    "build:dev": "npm run build:js-sdk && npm run build:react-sdk && npm run reskindex && npm run build:res && npm run build:bundle:dev",
     "dist": "scripts/package.sh",
     "install:electron": "install-app-deps",
     "electron": "npm run install:electron && electron .",
     "start:res": "node scripts/copy-res.js -w",
     "start:js": "webpack-dev-server --output-filename=bundles/_dev_/[name].js --output-chunk-file=bundles/_dev_/[name].js -w --progress",
     "start:js:prod": "cross-env NODE_ENV=production webpack-dev-server -w --progress",
-    "start": "parallelshell \"npm run reskindex:watch\" \"npm run start:res\" \"npm run start:js\"",
+    "start:js-sdk": "node scripts//build-watch-sdk.js watch js",
-    "start:prod": "parallelshell \"npm run reskindex:watch\" \"npm run start:res\" \"npm run start:js:prod\"",
+    "start:js-sdk:prod": "cross-env NODE_ENV=production node scripts//build-watch-sdk.js watch js",
+    "start:react-sdk": "node scripts//build-watch-sdk.js watch react",
+    "start:react-sdk:prod": "cross-env NODE_ENV=production node scripts//build-watch-sdk.js watch react",
+    "start": "concurrently --kill-others -n js-sdk,react-sdk,reskindex,res,js \"npm run start:js-sdk\" \"npm run start:react-sdk\" \"npm run reskindex:watch\" \"npm run start:res\" \"npm run start:js\"",
+    "start:prod": "concurrently --kill-others -n js-sdk,react-sdk,reskindex,res,js \"npm run start:js-sdk:prod\" \"npm run start:react-sdk:prod\" \"npm run reskindex:watch\" \"npm run start:res\" \"npm run start:js:prod\"",
     "lint": "eslint src/",
     "lintall": "eslint src/ test/",
     "clean": "rimraf lib webapp electron_app/dist",
@@ -91,6 +97,7 @@
     "babel-preset-react": "^6.16.0",
     "babel-preset-stage-2": "^6.17.0",
     "chokidar": "^1.6.1",
+    "concurrently": "^4.0.1",
     "cpx": "^1.3.2",
     "cross-env": "^4.0.0",
     "css-raw-loader": "^0.1.1",
@@ -122,7 +129,6 @@
     "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
     "mocha": "^2.4.5",
-    "parallelshell": "3.0.1",
     "postcss-extend": "^1.0.5",
     "postcss-import": "^9.0.0",
     "postcss-loader": "^1.2.2",

scripts/build-watch-sdk.js

@@ -0,0 +1,44 @@
+const fs = require('fs');
+const path = require('path');
+const child_process = require('child_process');
+
+
+const task = process.argv[2];
+if (task !== "build" && task !== "watch") {
+    console.error("Expected a task of 'build' or 'watch'");
+    process.exit(1);
+}
+
+const sdkName = process.argv[3];
+if (!sdkName) {
+    console.error("Missing SDK name");
+    process.exit(1);
+}
+
+const sdkPath = path.dirname(require.resolve(`matrix-${sdkName}-sdk/package.json`));
+console.log(sdkPath);
+
+// We only want to build the SDK if it looks like it was `npm link`ed
+if (fs.existsSync(path.join(sdkPath, '.git'))) {
+    console.log("Installing develop dependencies");
+    const devEnv = Object.assign({}, process.env, {NODE_ENV: "development"});
+    child_process.execSync("npm install --only=dev", {
+        env: devEnv,
+        cwd: sdkPath,
+    });
+
+    // Because webpack is made of fail
+    if (sdkName === "js") {
+        console.log("Installing source-map-loader");
+        child_process.execSync("npm install source-map-loader", {
+            env: devEnv,
+            cwd: sdkPath,
+        });
+    }
+
+    console.log("Performing task: " + task);
+    child_process.execSync(`npm ${task === "build" ? "run build" : "start"}`, {
+        env: process.env,
+        cwd: sdkPath,
+    });
+}

scripts/fetch-develop.deps.sh

@@ -52,13 +52,7 @@ function dodep() {
     echo "$repo set to branch "`git -C "$repo" rev-parse --abbrev-ref HEAD`
 
     mkdir -p node_modules
-    rm -r "node_modules/$repo" 2>/dev/null || true
+    npm link "../$repo"  # This does an npm install for us
-    ln -sv "../$repo" node_modules/
-
-    (
-        cd $repo
-        npm install
-    )
 }
 
 ##############################
@@ -77,14 +71,15 @@ echo 'Setting up matrix-react-sdk'
 
 dodep matrix-org matrix-react-sdk
 
-# replace the version of js-sdk that got pulled into react-sdk with a symlink
+# replace the version of js-sdk that got pulled into react-sdk with a link
 # to our version. Make sure to do this *after* doing 'npm i' in react-sdk,
 # otherwise npm helpfully moves another-json from matrix-js-sdk/node_modules
 # into matrix-react-sdk/node_modules.
 #
 # (note this matches the instructions in the README.)
-rm -r node_modules/matrix-react-sdk/node_modules/matrix-js-sdk
+cd matrix-react-sdk
-ln -s ../../matrix-js-sdk node_modules/matrix-react-sdk/node_modules/
+npm link ../matrix-js-sdk
+cd ../
 
 echo -en 'travis_fold:end:matrix-react-sdk\r'