Merge remote-tracking branch 'origin/develop' into develop

electron_app/src/electron-main.js

@@ -24,7 +24,7 @@ const checkSquirrelHooks = require('./squirrelhooks');
 if (checkSquirrelHooks()) return;
 
 const argv = require('minimist')(process.argv);
-const {app, ipcMain, powerSaveBlocker, BrowserWindow, Menu, autoUpdater} = require('electron');
+const {app, ipcMain, powerSaveBlocker, BrowserWindow, Menu, autoUpdater, protocol} = require('electron');
 const AutoLaunch = require('auto-launch');
 const path = require('path');
 
@@ -32,9 +32,15 @@ const tray = require('./tray');
 const vectorMenu = require('./vectormenu');
 const webContentsHandler = require('./webcontents-handler');
 const updater = require('./updater');
+const { migrateFromOldOrigin } = require('./originMigrator');
 
 const windowStateKeeper = require('electron-window-state');
 
+// boolean flag set whilst we are doing one-time origin migration
+// We only serve the origin migration script while we're actually
+// migrating to mitigate any risk of it being used maliciously.
+let migratingOrigin = false;
+
 if (argv['profile']) {
     app.setPath('userData', `${app.getPath('userData')}-${argv['profile']}`);
 }
@@ -110,7 +116,7 @@ autoUpdater.on('update-downloaded', (ev, releaseNotes, releaseName, releaseDate,
     });
 });
 
-ipcMain.on('ipcCall', function(ev, payload) {
+ipcMain.on('ipcCall', async function(ev, payload) {
     if (!mainWindow) return;
 
     const args = payload.args || [];
@@ -141,10 +147,15 @@ ipcMain.on('ipcCall', function(ev, payload) {
             } else {
                 mainWindow.focus();
             }
+        case 'origin_migrate':
+            migratingOrigin = true;
+            await migrateFromOldOrigin();
+            migratingOrigin = false;
+            break;
         default:
             mainWindow.webContents.send('ipcReply', {
                 id: payload.id,
-                error: new Error("Unknown IPC Call: "+payload.name),
+                error: "Unknown IPC Call: " + payload.name,
             });
             return;
     }
@@ -171,6 +182,13 @@ const launcher = new AutoLaunch({
     },
 });
 
+// Register the scheme the app is served from as 'standard'
+// which allows things like relative URLs and IndexedDB to
+// work.
+// Also mark it as secure (ie. accessing resources from this
+// protocol and HTTPS won't trigger mixed content warnings).
+protocol.registerStandardSchemes(['vector'], {secure: true});
+
 app.on('ready', () => {
     if (argv['devtools']) {
         try {
@@ -186,6 +204,66 @@ app.on('ready', () => {
         }
     }
 
+    protocol.registerFileProtocol('vector', (request, callback) => {
+        if (request.method !== 'GET') {
+            callback({error: -322}); // METHOD_NOT_SUPPORTED from chromium/src/net/base/net_error_list.h
+            return null;
+        }
+
+        const parsedUrl = new URL(request.url);
+        if (parsedUrl.protocol !== 'vector:') {
+            callback({error: -302}); // UNKNOWN_URL_SCHEME
+            return;
+        }
+        if (parsedUrl.host !== 'vector') {
+            callback({error: -105}); // NAME_NOT_RESOLVED
+            return;
+        }
+
+        const target = parsedUrl.pathname.split('/');
+
+        // path starts with a '/'
+        if (target[0] !== '') {
+            callback({error: -6}); // FILE_NOT_FOUND
+            return;
+        }
+
+        if (target[target.length - 1] == '') {
+            target[target.length - 1] = 'index.html';
+        }
+
+        let baseDir;
+        // first part of the path determines where we serve from
+        if (migratingOrigin && target[1] === 'origin_migrator_dest') {
+            // the origin migrator destination page
+            // (only the destination script needs to come from the
+            // custom protocol: the source part is loaded from a
+            // file:// as that's the origin we're migrating from).
+            baseDir = __dirname + "/../../origin_migrator/dest";
+        } else if (target[1] === 'webapp') {
+            baseDir = __dirname + "/../../webapp";
+        } else {
+            callback({error: -6}); // FILE_NOT_FOUND
+            return;
+        }
+
+        // Normalise the base dir and the target path separately, then make sure
+        // the target path isn't trying to back out beyond its root
+        baseDir = path.normalize(baseDir);
+
+        const relTarget = path.normalize(path.join(...target.slice(2)));
+        if (relTarget.startsWith('..')) {
+            callback({error: -6}); // FILE_NOT_FOUND
+            return;
+        }
+        const absTarget = path.join(baseDir, relTarget);
+
+        callback({
+            path: absTarget,
+        });
+    }, (error) => {
+        if (error) console.error('Failed to register protocol')
+    });
 
     if (vectorConfig['update_base_url']) {
         console.log(`Starting auto update with base URL: ${vectorConfig['update_base_url']}`);
@@ -225,7 +303,7 @@ app.on('ready', () => {
             webgl: false,
         },
     });
-    mainWindow.loadURL(`file://${__dirname}/../../webapp/index.html`);
+    mainWindow.loadURL('vector://vector/webapp/');
     Menu.setApplicationMenu(vectorMenu);
 
     // explicitly hide because setApplicationMenu on Linux otherwise shows...

electron_app/src/originMigrator.js

@@ -0,0 +1,62 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const { BrowserWindow, ipcMain } = require('electron');
+const path = require('path');
+
+async function migrateFromOldOrigin() {
+    console.log("Attempting to migrate data between origins");
+
+    // We can use the same preload script: we just need ipcRenderer exposed
+    const preloadScript = path.normalize(`${__dirname}/preload.js`);
+    await new Promise(resolve => {
+        const migrateWindow = new BrowserWindow({
+            show: false,
+            webPreferences: {
+                preload: preloadScript,
+                nodeIntegration: false,
+                sandbox: true,
+                enableRemoteModule: false,
+                webgl: false,
+            },
+        });
+        ipcMain.on('origin_migration_complete', (e, success, sentSummary, storedSummary) => {
+            if (success) {
+                console.log("Origin migration completed successfully!");
+            } else {
+                console.error("Origin migration failed!");
+            }
+            console.error("Data sent", sentSummary);
+            console.error("Data stored", storedSummary);
+            migrateWindow.close();
+            resolve();
+        });
+        ipcMain.on('origin_migration_nodata', (e) => {
+            console.log("No session to migrate from old origin");
+            migrateWindow.close();
+            resolve();
+        });
+        // Normalise the path because in the distribution, __dirname will be inside the
+        // electron asar.
+        const sourcePagePath = path.normalize(__dirname + '/../../origin_migrator/source.html');
+        console.log("Loading path: " + sourcePagePath);
+        migrateWindow.loadURL('file://' + sourcePagePath);
+    });
+}
+
+module.exports = {
+    migrateFromOldOrigin,
+};

electron_app/src/preload.js

@@ -14,8 +14,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-const { ipcRenderer } = require('electron');
+const { ipcRenderer, webFrame } = require('electron');
 
 // expose ipcRenderer to the renderer process
 window.ipcRenderer = ipcRenderer;
 
+// Allow the fetch API to load resources from this
+// protocol: this is necessary to load olm.wasm.
+// (Also mark it a secure although we've already
+// done this in the main process).
+webFrame.registerURLSchemeAsPrivileged('vector', {
+    secure: true,
+    supportFetchAPI: true,
+});

origin_migrator/dest/dest.html

@@ -0,0 +1,6 @@
+<html>
+<body>
+<script src="browser-matrix.min.js"></script>
+<script src="dest.js"></script>
+</body>
+</html>

origin_migrator/dest/dest.js

@@ -0,0 +1,125 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const SOURCE_ORIGIN = 'file://';
+
+const IndexedDBCryptoStore = window.matrixcs.IndexedDBCryptoStore;
+const cryptoStore = new IndexedDBCryptoStore(window.indexedDB, 'matrix-js-sdk:crypto');
+
+let accountStored = 0;
+let sessionsStored = 0;
+let inboundGroupSessionsStored = 0;
+let deviceDataStored = 0;
+let roomsStored = 0;
+let localStorageKeysStored = 0;
+
+const promises = [];
+
+async function onMessage(e) {
+    if (e.origin !== SOURCE_ORIGIN) return;
+
+    const data = e.data.data; // bleh, naming clash
+    switch (e.data.cmd) {
+        case 'init':
+            // start with clean stores before we migrate data in
+            window.localStorage.clear();
+            await cryptoStore.deleteAllData();
+
+            e.source.postMessage({
+                cmd: 'initOK',
+            }, SOURCE_ORIGIN);
+            break;
+        case 'storeAccount':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    cryptoStore.storeAccount(txn, data);
+                },
+            ).then(() => {
+                ++accountStored;
+            }));
+            break;
+        case 'storeSessions':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_SESSIONS],
+                (txn) => {
+                    for (const sess of data) {
+                        cryptoStore.storeEndToEndSession(sess.deviceKey, sess.sessionId, sess, txn);
+                    }
+                },
+            ).then(() => {
+                sessionsStored += data.length;
+            }));
+            break;
+        case 'storeInboundGroupSessions':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+                (txn) => {
+                    for (const sess of data) {
+                        cryptoStore.addEndToEndInboundGroupSession(
+                            sess.senderKey, sess.sessionId, sess.sessionData, txn,
+                        );
+                    }
+                },
+            ).then(() => {
+                inboundGroupSessionsStored += data.length;
+            }));
+            break;
+        case 'storeDeviceData':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA],
+                (txn) => {
+                    cryptoStore.storeEndToEndDeviceData(data, txn);
+                },
+            ).then(() => {
+                ++deviceDataStored;
+            }));
+            break;
+        case 'storeRooms':
+            promises.push(cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ROOMS],
+                (txn) => {
+                    for (const [roomId, roomInfo] of Object.entries(data)) {
+                        cryptoStore.storeEndToEndRoom(roomId, roomInfo, txn);
+                    }
+                },
+            ).then(() => {
+                ++roomsStored;
+            }));
+            break;
+        case 'storeLocalStorage':
+            window.localStorage.setItem(data.key, data.val);
+            ++localStorageKeysStored;
+            break;
+        case 'getSummary':
+            await Promise.all(promises);
+            e.source.postMessage({
+                cmd: 'summary',
+                data: {
+                    accountStored,
+                    sessionsStored,
+                    inboundGroupSessionsStored,
+                    deviceDataStored,
+                    roomsStored,
+                    localStorageKeysStored,
+                },
+            }, SOURCE_ORIGIN);
+            break;
+    }
+}
+
+window.addEventListener('message', onMessage);
+

origin_migrator/source.html

@@ -0,0 +1,7 @@
+<html>
+<body>
+<script src="dest/browser-matrix.min.js"></script>
+<script src="source.js"></script>
+<iframe name="dest" src="vector://vector/origin_migrator_dest/dest.html" onload="doMigrate()"></iframe>
+</body>
+</html>

origin_migrator/source.js

@@ -0,0 +1,210 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const TARGET_ORIGIN = 'vector://vector';
+const BATCH_SIZE = 500;
+let destFrame;
+
+let initResolver = null;
+let getSummaryResolver = null;
+
+function onMessage(e) {
+    if (e.origin !== TARGET_ORIGIN) return;
+
+    if (e.data.cmd === 'initOK' && initResolver) {
+        initResolver();
+        initResolver = null;
+    } else if (e.data.cmd === 'summary' && getSummaryResolver) {
+        getSummaryResolver(e.data.data);
+        getSummaryResolver = null;
+    }
+}
+
+async function initDestFrame() {
+    return new Promise(resolve => {
+        initResolver = resolve;
+        destFrame.postMessage({
+            cmd: 'init', 
+        }, TARGET_ORIGIN);
+    });
+}
+
+async function getSummary() {
+    return new Promise(resolve => {
+        getSummaryResolver = resolve;
+        destFrame.postMessage({
+            cmd: 'getSummary', 
+        }, TARGET_ORIGIN);
+    });
+}
+
+async function doMigrate() {
+    let accountSent = 0;
+    let sessionsSent = 0;
+    let inboundGroupSessionsSent = 0;
+    let deviceDataSent = 0;
+    let roomsSent = 0;
+    let localStorageKeysSent = 0;
+
+    if (!window.ipcRenderer) {
+        console.error("ipcRenderer not found");
+        return;
+    }
+
+    if (window.localStorage.getItem('mx_user_id') === null) {
+        window.ipcRenderer.send("origin_migration_nodata");
+        return;
+    }
+
+    destFrame = window.parent.frames.dest;
+
+    await initDestFrame();
+
+    const IndexedDBCryptoStore = window.matrixcs.IndexedDBCryptoStore;
+
+    const cryptoStore = new IndexedDBCryptoStore(window.indexedDB, 'matrix-js-sdk:crypto');
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            cryptoStore.getAccount(txn, (account) => {
+                destFrame.postMessage({
+                    cmd: 'storeAccount',
+                    data: account,
+                }, TARGET_ORIGIN);
+                ++accountSent;
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_SESSIONS],
+        (txn) => {
+            let sessBatch = [];
+            cryptoStore.getAllEndToEndSessions(txn, (sessInfo) => {
+                if (sessInfo) {
+                    ++sessionsSent;
+                    sessBatch.push(sessInfo);
+                }
+                if (sessBatch.length >= BATCH_SIZE || sessInfo === null) {
+                    destFrame.postMessage({
+                        cmd: 'storeSessions',
+                        data: sessBatch,
+                    }, TARGET_ORIGIN);
+                    sessBatch = [];
+                }
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+        (txn) => {
+            let sessBatch = [];
+            cryptoStore.getAllEndToEndInboundGroupSessions(txn, (sessInfo) => {
+                if (sessInfo) {
+                    ++inboundGroupSessionsSent;
+                    sessBatch.push(sessInfo);
+                }
+                if (sessBatch.length >= BATCH_SIZE || sessInfo === null) {
+                    destFrame.postMessage({
+                        cmd: 'storeInboundGroupSessions',
+                        data: sessBatch,
+                    }, TARGET_ORIGIN);
+                    sessBatch = [];
+                }
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA],
+        (txn) => {
+            cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                destFrame.postMessage({
+                    cmd: 'storeDeviceData',
+                    data: deviceData,
+                }, TARGET_ORIGIN);
+                ++deviceDataSent;
+            });
+        },
+    );
+
+    await cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ROOMS],
+        (txn) => {
+            cryptoStore.getEndToEndRooms(txn, (rooms) => {
+                destFrame.postMessage({
+                    cmd: 'storeRooms',
+                    data: rooms,
+                }, TARGET_ORIGIN);
+                ++roomsSent;
+            });
+        },
+    );
+
+    // we don't bother migrating;
+    // * sync data (we can just initialsync again)
+    // * logs
+    // * key requests (worst case they'll just be re-sent)
+    // * sessions needing backup (feature isn't available on Electron)
+
+    for (let i = 0; i < window.localStorage.length; ++i) {
+        const key = window.localStorage.key(i);
+        const val = window.localStorage.getItem(key);
+        
+        destFrame.postMessage({
+            cmd: 'storeLocalStorage',
+            data: { key, val },
+        }, TARGET_ORIGIN);
+        ++localStorageKeysSent;
+    }
+
+    const summary = await getSummary();
+    let success = false;
+    if (
+        summary.accountStored === accountSent &&
+        summary.sessionsStored === sessionsSent &&
+        summary.inboundGroupSessionsStored === inboundGroupSessionsSent &&
+        summary.deviceDataStored === deviceDataSent &&
+        summary.roomsStored === roomsSent &&
+        summary.localStorageKeysStored === localStorageKeysSent
+    ) {
+        success = true;
+        window.localStorage.clear();
+        await cryptoStore.deleteAllData();
+
+        // we don't bother migrating them, but also blow away the sync & logs db,
+        // otherwise they'll just hang about taking up space
+        await new Promise(resolve => {
+            const req = window.indexedDB.deleteDatabase('matrix-js-sdk:riot-web-sync');
+            req.onsuccess = resolve;
+            req.onerror = resolve;
+        });
+        await new Promise(resolve => {
+            const req = window.indexedDB.deleteDatabase('logs');
+            req.onsuccess = resolve;
+            req.onerror = resolve;
+        });
+    }
+
+    window.ipcRenderer.send("origin_migration_complete", success, {
+        accountSent, sessionsSent, inboundGroupSessionsSent,
+        deviceDataSent, roomsSent, localStorageKeysSent,
+    }, summary);
+}
+
+window.addEventListener('message', onMessage);

package.json

@@ -157,7 +157,8 @@
       "img/**"
     ],
     "extraResources": [
-      "webapp/**/*"
+      "webapp/**/*",
+      "origin_migrator/**/*"
     ],
     "linux": {
       "target": "deb",

src/vector/index.js

@@ -231,7 +231,16 @@ async function loadApp() {
     // set the platform for react sdk
     if (window.ipcRenderer) {
         console.log("Using Electron platform");
-        PlatformPeg.set(new ElectronPlatform());
+        const plaf = new ElectronPlatform();
+        PlatformPeg.set(plaf);
+
+        // Electron only: see if we need to do a one-time data
+        // migration
+        if (window.localStorage.getItem('mx_user_id') === null) {
+            console.log("Migrating session from old origin...");
+            await plaf.migrateFromOldOrigin();
+            console.log("Origin migration complete");
+        }
     } else {
         console.log("Using Web platform");
         PlatformPeg.set(new WebPlatform());

src/vector/platform/ElectronPlatform.js

@@ -224,6 +224,10 @@ export default class ElectronPlatform extends VectorBasePlatform {
         window.location.reload(false);
     }
 
+    async migrateFromOldOrigin() {
+        return this._ipcCall('origin_migrate');
+    }
+
     async _ipcCall(name, ...args) {
         const ipcCallId = ++this._nextIpcCallId;
         return new Promise((resolve, reject) => {

src/vector/platform/VectorBasePlatform.js

@@ -149,4 +149,12 @@ export default class VectorBasePlatform extends BasePlatform {
     getDefaultDeviceDisplayName(): string {
         return _t("Unknown device");
     }
+
+    /**
+     * Migrate account data from a previous origin
+     * Used only for the electron app
+     */
+    async migrateFromOldOrigin() {
+        return false;
+    }
 }