first cut at embedding dendrite into riot as a service-worker

package.json

@@ -64,6 +64,8 @@
     "favico.js": "^0.3.10",
     "gemini-scrollbar": "github:matrix-org/gemini-scrollbar#91e1e566",
     "gfm.css": "^1.1.2",
+    "go-sqlite-js": "^0.1.0",
+    "go-http-js-libp2p": "^0.1.0",
     "highlight.js": "^9.13.1",
     "matrix-js-sdk": "3.0.0",
     "matrix-react-sdk": "1.7.6",

src/vector/dendrite-sw.js

@@ -0,0 +1,53 @@
+// -*- coding: utf-8 -*-
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+// Copyright 2020 New Vector Ltd
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+self.importScripts("wasm_exec.js", "bundles/go_http_bridge.js", "bundles/sqlite_bridge.js")
+
+self.addEventListener('install', function(event) {
+    console.log("installing SW")
+})
+
+self.addEventListener('activate', function(event) {
+    console.log("SW activated")
+
+    const config = {
+        locateFile: filename => 'sql_wasm.wasm'
+    }
+
+    event.waitUntil(
+        sqlite_bridge.init(config).then(()=>{
+            const go = new Go()
+            WebAssembly.instantiateStreaming(fetch('dendrite.wasm'), go.importObject).then((result) => {
+                go.run(result.instance)
+            });
+        })
+    )
+})
+
+self.addEventListener('fetch', function(event) {
+    console.log("intercepted " + event.request.url)
+    if (event.request.url.match(/\/_matrix\/client/)) {
+        if (global.fetchListener) {
+            event.respondWith(global.fetchListener.onFetch(event))
+        }
+        else {
+            console.log("no fetch listener present for " + event.request.url)
+        }
+    }
+    else {
+        return fetch(event.request)
+    }
+})

src/vector/index.html

@@ -39,6 +39,7 @@
     <noscript>Sorry, Riot requires JavaScript to be enabled.</noscript> <!-- TODO: Translate this? -->
     <script>
         window.vector_indexeddb_worker_script = '<%= htmlWebpackPlugin.files.chunks['indexeddb-worker'].entry %>';
+        window.vector_dendrite_worker_script = '<%= htmlWebpackPlugin.files.chunks['dendrite-sw'].entry %>';
     </script>
     <script src="<%= htmlWebpackPlugin.files.chunks['bundle'].entry %>"></script>
     <script>

src/vector/index.js

@@ -180,6 +180,20 @@ async function loadApp() {
         throw new Error("Missing indexeddb worker script!");
     }
     MatrixClientPeg.setIndexedDbWorkerScript(window.vector_indexeddb_worker_script);
+
+    // load dendrite, if available
+    if (window.vector_dendrite_worker_script && 'serviceWorker' in navigator) {
+        window.addEventListener('load', ()=>{
+            navigator.serviceWorker.register(window.vector_dendrite_worker_script).then(function(registration) {
+                // Registration was successful
+                console.log('ServiceWorker registration successful with scope: ', registration.scope)
+            }, (err)=>{
+                // registration failed :(
+                console.log('ServiceWorker registration failed: ', err)
+            })
+        })
+    }    
+
     CallHandler.setConferenceHandler(VectorConferenceHandler);
 
     window.addEventListener('hashchange', onHashChange);

src/vector/wasm_exec.js

@@ -0,0 +1,533 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+(() => {
+	// Map multiple JavaScript environments to a single common API,
+	// preferring web standards over Node.js API.
+	//
+	// Environments considered:
+	// - Browsers
+	// - Node.js
+	// - Electron
+	// - Parcel
+
+	if (typeof global !== "undefined") {
+		// global already exists
+	} else if (typeof window !== "undefined") {
+		window.global = window;
+	} else if (typeof self !== "undefined") {
+		self.global = self;
+	} else {
+		throw new Error("cannot export Go (neither global, window nor self is defined)");
+	}
+
+	if (!global.require && typeof require !== "undefined") {
+		global.require = require;
+	}
+
+	if (!global.fs && global.require) {
+		global.fs = require("fs");
+	}
+
+	if (!global.fs) {
+		let outputBuf = "";
+		global.fs = {
+			constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused
+			writeSync(fd, buf) {
+				outputBuf += decoder.decode(buf);
+				const nl = outputBuf.lastIndexOf("\n");
+				if (nl != -1) {
+					console.log(outputBuf.substr(0, nl));
+					outputBuf = outputBuf.substr(nl + 1);
+				}
+				return buf.length;
+			},
+			write(fd, buf, offset, length, position, callback) {
+				if (offset !== 0 || length !== buf.length || position !== null) {
+					throw new Error("not implemented");
+				}
+				const n = this.writeSync(fd, buf);
+				callback(null, n);
+			},
+			open(path, flags, mode, callback) {
+				const err = new Error("not implemented");
+				err.code = "ENOSYS";
+				callback(err);
+			},
+			read(fd, buffer, offset, length, position, callback) {
+				const err = new Error("not implemented");
+				err.code = "ENOSYS";
+				callback(err);
+			},
+			fsync(fd, callback) {
+				callback(null);
+			},
+		};
+	}
+
+	if (!global.crypto) {
+		const nodeCrypto = require("crypto");
+		global.crypto = {
+			getRandomValues(b) {
+				nodeCrypto.randomFillSync(b);
+			},
+		};
+	}
+
+	if (!global.performance) {
+		global.performance = {
+			now() {
+				const [sec, nsec] = process.hrtime();
+				return sec * 1000 + nsec / 1000000;
+			},
+		};
+	}
+
+	if (!global.TextEncoder) {
+		global.TextEncoder = require("util").TextEncoder;
+	}
+
+	if (!global.TextDecoder) {
+		global.TextDecoder = require("util").TextDecoder;
+	}
+
+	// End of polyfills for common API.
+
+	const encoder = new TextEncoder("utf-8");
+	const decoder = new TextDecoder("utf-8");
+
+	global.Go = class {
+		constructor() {
+			this.argv = ["js"];
+			this.env = {};
+			this.exit = (code) => {
+				if (code !== 0) {
+					console.warn("exit code:", code);
+				}
+			};
+			this._exitPromise = new Promise((resolve) => {
+				this._resolveExitPromise = resolve;
+			});
+			this._pendingEvent = null;
+			this._scheduledTimeouts = new Map();
+			this._nextCallbackTimeoutID = 1;
+
+			const mem = () => {
+				// The buffer may change when requesting more memory.
+				return new DataView(this._inst.exports.mem.buffer);
+			}
+
+			const setInt64 = (addr, v) => {
+				mem().setUint32(addr + 0, v, true);
+				mem().setUint32(addr + 4, Math.floor(v / 4294967296), true);
+			}
+
+			const getInt64 = (addr) => {
+				const low = mem().getUint32(addr + 0, true);
+				const high = mem().getInt32(addr + 4, true);
+				return low + high * 4294967296;
+			}
+
+			const loadValue = (addr) => {
+				const f = mem().getFloat64(addr, true);
+				if (f === 0) {
+					return undefined;
+				}
+				if (!isNaN(f)) {
+					return f;
+				}
+
+				const id = mem().getUint32(addr, true);
+				return this._values[id];
+			}
+
+			const storeValue = (addr, v) => {
+				const nanHead = 0x7FF80000;
+
+				if (typeof v === "number") {
+					if (isNaN(v)) {
+						mem().setUint32(addr + 4, nanHead, true);
+						mem().setUint32(addr, 0, true);
+						return;
+					}
+					if (v === 0) {
+						mem().setUint32(addr + 4, nanHead, true);
+						mem().setUint32(addr, 1, true);
+						return;
+					}
+					mem().setFloat64(addr, v, true);
+					return;
+				}
+
+				switch (v) {
+					case undefined:
+						mem().setFloat64(addr, 0, true);
+						return;
+					case null:
+						mem().setUint32(addr + 4, nanHead, true);
+						mem().setUint32(addr, 2, true);
+						return;
+					case true:
+						mem().setUint32(addr + 4, nanHead, true);
+						mem().setUint32(addr, 3, true);
+						return;
+					case false:
+						mem().setUint32(addr + 4, nanHead, true);
+						mem().setUint32(addr, 4, true);
+						return;
+				}
+
+				let ref = this._refs.get(v);
+				if (ref === undefined) {
+					ref = this._values.length;
+					this._values.push(v);
+					this._refs.set(v, ref);
+				}
+				let typeFlag = 0;
+				switch (typeof v) {
+					case "string":
+						typeFlag = 1;
+						break;
+					case "symbol":
+						typeFlag = 2;
+						break;
+					case "function":
+						typeFlag = 3;
+						break;
+				}
+				mem().setUint32(addr + 4, nanHead | typeFlag, true);
+				mem().setUint32(addr, ref, true);
+			}
+
+			const loadSlice = (addr) => {
+				const array = getInt64(addr + 0);
+				const len = getInt64(addr + 8);
+				return new Uint8Array(this._inst.exports.mem.buffer, array, len);
+			}
+
+			const loadSliceOfValues = (addr) => {
+				const array = getInt64(addr + 0);
+				const len = getInt64(addr + 8);
+				const a = new Array(len);
+				for (let i = 0; i < len; i++) {
+					a[i] = loadValue(array + i * 8);
+				}
+				return a;
+			}
+
+			const loadString = (addr) => {
+				const saddr = getInt64(addr + 0);
+				const len = getInt64(addr + 8);
+				return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len));
+			}
+
+			const timeOrigin = Date.now() - performance.now();
+			this.importObject = {
+				go: {
+					// Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters)
+					// may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported
+					// function. A goroutine can switch to a new stack if the current stack is too small (see morestack function).
+					// This changes the SP, thus we have to update the SP used by the imported function.
+
+					// func wasmExit(code int32)
+					"runtime.wasmExit": (sp) => {
+						const code = mem().getInt32(sp + 8, true);
+						this.exited = true;
+						delete this._inst;
+						delete this._values;
+						delete this._refs;
+						this.exit(code);
+					},
+
+					// func wasmWrite(fd uintptr, p unsafe.Pointer, n int32)
+					"runtime.wasmWrite": (sp) => {
+						const fd = getInt64(sp + 8);
+						const p = getInt64(sp + 16);
+						const n = mem().getInt32(sp + 24, true);
+						fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n));
+					},
+
+					// func nanotime() int64
+					"runtime.nanotime": (sp) => {
+						setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000);
+					},
+
+					// func walltime() (sec int64, nsec int32)
+					"runtime.walltime": (sp) => {
+						const msec = (new Date).getTime();
+						setInt64(sp + 8, msec / 1000);
+						mem().setInt32(sp + 16, (msec % 1000) * 1000000, true);
+					},
+
+					// func scheduleTimeoutEvent(delay int64) int32
+					"runtime.scheduleTimeoutEvent": (sp) => {
+						const id = this._nextCallbackTimeoutID;
+						this._nextCallbackTimeoutID++;
+						this._scheduledTimeouts.set(id, setTimeout(
+							() => {
+								this._resume();
+								while (this._scheduledTimeouts.has(id)) {
+									// for some reason Go failed to register the timeout event, log and try again
+									// (temporary workaround for https://github.com/golang/go/issues/28975)
+									console.warn("scheduleTimeoutEvent: missed timeout event");
+									this._resume();
+								}
+							},
+							getInt64(sp + 8) + 1, // setTimeout has been seen to fire up to 1 millisecond early
+						));
+						mem().setInt32(sp + 16, id, true);
+					},
+
+					// func clearTimeoutEvent(id int32)
+					"runtime.clearTimeoutEvent": (sp) => {
+						const id = mem().getInt32(sp + 8, true);
+						clearTimeout(this._scheduledTimeouts.get(id));
+						this._scheduledTimeouts.delete(id);
+					},
+
+					// func getRandomData(r []byte)
+					"runtime.getRandomData": (sp) => {
+						crypto.getRandomValues(loadSlice(sp + 8));
+					},
+
+					// func stringVal(value string) ref
+					"syscall/js.stringVal": (sp) => {
+						storeValue(sp + 24, loadString(sp + 8));
+					},
+
+					// func valueGet(v ref, p string) ref
+					"syscall/js.valueGet": (sp) => {
+						const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16));
+						sp = this._inst.exports.getsp(); // see comment above
+						storeValue(sp + 32, result);
+					},
+
+					// func valueSet(v ref, p string, x ref)
+					"syscall/js.valueSet": (sp) => {
+						Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32));
+					},
+
+					// func valueIndex(v ref, i int) ref
+					"syscall/js.valueIndex": (sp) => {
+						storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16)));
+					},
+
+					// valueSetIndex(v ref, i int, x ref)
+					"syscall/js.valueSetIndex": (sp) => {
+						Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24));
+					},
+
+					// func valueCall(v ref, m string, args []ref) (ref, bool)
+					"syscall/js.valueCall": (sp) => {
+						try {
+							const v = loadValue(sp + 8);
+							const m = Reflect.get(v, loadString(sp + 16));
+							const args = loadSliceOfValues(sp + 32);
+							const result = Reflect.apply(m, v, args);
+							sp = this._inst.exports.getsp(); // see comment above
+							storeValue(sp + 56, result);
+							mem().setUint8(sp + 64, 1);
+						} catch (err) {
+							storeValue(sp + 56, err);
+							mem().setUint8(sp + 64, 0);
+						}
+					},
+
+					// func valueInvoke(v ref, args []ref) (ref, bool)
+					"syscall/js.valueInvoke": (sp) => {
+						try {
+							const v = loadValue(sp + 8);
+							const args = loadSliceOfValues(sp + 16);
+							const result = Reflect.apply(v, undefined, args);
+							sp = this._inst.exports.getsp(); // see comment above
+							storeValue(sp + 40, result);
+							mem().setUint8(sp + 48, 1);
+						} catch (err) {
+							storeValue(sp + 40, err);
+							mem().setUint8(sp + 48, 0);
+						}
+					},
+
+					// func valueNew(v ref, args []ref) (ref, bool)
+					"syscall/js.valueNew": (sp) => {
+						try {
+							const v = loadValue(sp + 8);
+							const args = loadSliceOfValues(sp + 16);
+							const result = Reflect.construct(v, args);
+							sp = this._inst.exports.getsp(); // see comment above
+							storeValue(sp + 40, result);
+							mem().setUint8(sp + 48, 1);
+						} catch (err) {
+							storeValue(sp + 40, err);
+							mem().setUint8(sp + 48, 0);
+						}
+					},
+
+					// func valueLength(v ref) int
+					"syscall/js.valueLength": (sp) => {
+						setInt64(sp + 16, parseInt(loadValue(sp + 8).length));
+					},
+
+					// valuePrepareString(v ref) (ref, int)
+					"syscall/js.valuePrepareString": (sp) => {
+						const str = encoder.encode(String(loadValue(sp + 8)));
+						storeValue(sp + 16, str);
+						setInt64(sp + 24, str.length);
+					},
+
+					// valueLoadString(v ref, b []byte)
+					"syscall/js.valueLoadString": (sp) => {
+						const str = loadValue(sp + 8);
+						loadSlice(sp + 16).set(str);
+					},
+
+					// func valueInstanceOf(v ref, t ref) bool
+					"syscall/js.valueInstanceOf": (sp) => {
+						mem().setUint8(sp + 24, loadValue(sp + 8) instanceof loadValue(sp + 16));
+					},
+
+					// func copyBytesToGo(dst []byte, src ref) (int, bool)
+					"syscall/js.copyBytesToGo": (sp) => {
+						const dst = loadSlice(sp + 8);
+						const src = loadValue(sp + 32);
+						if (!(src instanceof Uint8Array)) {
+							mem().setUint8(sp + 48, 0);
+							return;
+						}
+						const toCopy = src.subarray(0, dst.length);
+						dst.set(toCopy);
+						setInt64(sp + 40, toCopy.length);
+						mem().setUint8(sp + 48, 1);
+					},
+
+					// func copyBytesToJS(dst ref, src []byte) (int, bool)
+					"syscall/js.copyBytesToJS": (sp) => {
+						const dst = loadValue(sp + 8);
+						const src = loadSlice(sp + 16);
+						if (!(dst instanceof Uint8Array)) {
+							mem().setUint8(sp + 48, 0);
+							return;
+						}
+						const toCopy = src.subarray(0, dst.length);
+						dst.set(toCopy);
+						setInt64(sp + 40, toCopy.length);
+						mem().setUint8(sp + 48, 1);
+					},
+
+					"debug": (value) => {
+						console.log(value);
+					},
+				}
+			};
+		}
+
+		async run(instance) {
+			this._inst = instance;
+			this._values = [ // TODO: garbage collection
+				NaN,
+				0,
+				null,
+				true,
+				false,
+				global,
+				this,
+			];
+			this._refs = new Map();
+			this.exited = false;
+
+			const mem = new DataView(this._inst.exports.mem.buffer)
+
+			// Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory.
+			let offset = 4096;
+
+			const strPtr = (str) => {
+				const ptr = offset;
+				const bytes = encoder.encode(str + "\0");
+				new Uint8Array(mem.buffer, offset, bytes.length).set(bytes);
+				offset += bytes.length;
+				if (offset % 8 !== 0) {
+					offset += 8 - (offset % 8);
+				}
+				return ptr;
+			};
+
+			const argc = this.argv.length;
+
+			const argvPtrs = [];
+			this.argv.forEach((arg) => {
+				argvPtrs.push(strPtr(arg));
+			});
+
+			const keys = Object.keys(this.env).sort();
+			argvPtrs.push(keys.length);
+			keys.forEach((key) => {
+				argvPtrs.push(strPtr(`${key}=${this.env[key]}`));
+			});
+
+			const argv = offset;
+			argvPtrs.forEach((ptr) => {
+				mem.setUint32(offset, ptr, true);
+				mem.setUint32(offset + 4, 0, true);
+				offset += 8;
+			});
+
+			this._inst.exports.run(argc, argv);
+			if (this.exited) {
+				this._resolveExitPromise();
+			}
+			await this._exitPromise;
+		}
+
+		_resume() {
+			if (this.exited) {
+				throw new Error("Go program has already exited");
+			}
+			this._inst.exports.resume();
+			if (this.exited) {
+				this._resolveExitPromise();
+			}
+		}
+
+		_makeFuncWrapper(id) {
+			const go = this;
+			return function () {
+				const event = { id: id, this: this, args: arguments };
+				go._pendingEvent = event;
+				go._resume();
+				return event.result;
+			};
+		}
+	}
+
+	if (
+		global.require &&
+		global.require.main === module &&
+		global.process &&
+		global.process.versions &&
+		!global.process.versions.electron
+	) {
+		if (process.argv.length < 3) {
+			console.error("usage: go_js_wasm_exec [wasm binary] [arguments]");
+			process.exit(1);
+		}
+
+		const go = new Go();
+		go.argv = process.argv.slice(2);
+		go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env);
+		go.exit = process.exit;
+		WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => {
+			process.on("exit", (code) => { // Node.js exits if no event handler is pending
+				if (code === 0 && !go.exited) {
+					// deadlock, make Go print error and stack traces
+					go._pendingEvent = { id: 0 };
+					go._resume();
+				}
+			});
+			return go.run(result.instance);
+		}).catch((err) => {
+			console.error(err);
+			process.exit(1);
+		});
+	}
+})();

webpack.config.js

@@ -27,7 +27,12 @@ module.exports = (env, argv) => {
         entry: {
             "bundle": "./src/vector/index.js",
             "indexeddb-worker": "./src/vector/indexeddb-worker.js",
+            "dendrite-sw": "./src/vector/dendrite-sw.js",
             "mobileguide": "./src/vector/mobile_guide/index.js",
+            "sqlite_bridge": "./node_modules/go-sqlite3-js/js/bridge.js",
+            "go_http_bridge": "./node_modules/go-http-js-libp2p/js/bridge.js",
+            "sql_wasm": "./node_modules/go-sqlite3-js/node_modules/sql.js/dist/sql-wasm.wasm",
+            "dendrite_wasm": "./src/vector/dendrite.wasm",
 
             // CSS themes
             "theme-light": "./node_modules/matrix-react-sdk/res/themes/light/css/light.scss",
@@ -205,7 +210,9 @@ module.exports = (env, argv) => {
                     loader: "file-loader",
                     type: "javascript/auto", // https://github.com/webpack/webpack/issues/6725
                     options: {
-                        name: '[name].[hash:7].[ext]',
+                        // fixme - we should have a hash in this for cachebusting, but haven't figured
+                        // out a way yet to pass the resulting path from webpack into dendrite-sw
+                        name: '[name].[ext]',
                         outputPath: '.',
                     },
                 },
@@ -259,6 +266,10 @@ module.exports = (env, argv) => {
             ]
         },
 
+        node: {
+            fs: 'empty' // needed for sql.js
+        },
+
         plugins: [
             new webpack.DefinePlugin({
                 'process.env': {
@@ -308,6 +319,8 @@ module.exports = (env, argv) => {
             // chunks even after the app is redeployed.
             filename: "bundles/[hash]/[name].js",
             chunkFilename: "bundles/[hash]/[name].js",
+            libraryTarget: "var",
+            library: "[name]",
         },
 
         // configuration for the webpack-dev-server