Merge branch 'poljar/seshat-pr' into develop

docs/labs.md

@@ -73,3 +73,11 @@ improves the device verification experience by allowing you to verify a user
 instead of verifying each of their devices.
 
 This feature is still in development and will be landing in several chunks.
+
+## Event indexing and E2EE search support using Seshat (`feature_event_indexing`)
+
+Adds support for search in E2E encrypted rooms. This enables an event indexer
+that downloads, stores, and indexes room messages for E2E encrypted rooms.
+
+The existing search will transparently work for encrypted rooms just like it
+does for non-encrypted.

docs/native-node-modules.md

@@ -0,0 +1,59 @@
+# Native Node Modules
+
+For some features, the desktop version of Riot can make use of native Node
+modules. These allow Riot to integrate with the desktop in ways that a browser
+cannot.
+
+While native modules enable powerful new features, they must be complied for
+each operating system. For official Riot releases, we will always build these
+modules from source to ensure we can trust the compiled output. In the future,
+we may offer a pre-compiled path for those who want to use these features in a
+custom build of Riot without installing the various build tools required.
+
+Do note that compiling a module for a particular operating system
+(Linux/macOS/Windows) will need to be done on that operating system.
+Cross-compiling from a host OS for a different target OS may be possible, but
+we don't support this flow with Riot dependencies at this time.
+
+At the moment, we need to make some changes to the Riot release process before
+we can support native Node modules at release time, so these features are
+currently disabled by default until that is resolved. The following sections
+explain the manual steps you can use with a custom build of Riot to enable
+these features if you'd like to try them out.
+
+## Adding Seshat for search in E2E encrypted rooms
+
+Seshat is a native Node module that adds support for local event indexing and
+full text search in E2E encrypted rooms.
+
+Since Seshat is written in Rust, the Rust compiler and related tools need to be
+installed before installing Seshat itself. To install Rust please consult the
+official Rust [documentation](https://www.rust-lang.org/tools/install).
+
+Seshat also depends on the SQLCipher library to store its data in encrypted form
+on disk. You'll need to install it via your OS package manager.
+
+After installing the Rust compiler and SQLCipher, Seshat support can be added
+using yarn inside the `electron_app/` directory:
+
+    yarn add matrix-seshat
+
+After this is done the Electron version of Riot can be run from the main folder
+as usual using:
+
+    yarn electron
+
+If for some reason recompilation of Seshat is needed, e.g. when using a
+development version of Seshat using `yarn link`, or if the initial compilation was
+done for the wrong electron version, Seshat can be recompiled with the
+`electron-build-env` tool. Again from the `electron_app/` directory:
+
+    yarn add electron-build-env
+
+Recompiling Seshat itself can be done like so:
+
+    yarn run electron-build-env -- --electron 6.1.1 -- neon build matrix-seshat --release`
+
+Please make sure to include all the `--` as well as the `--release` command line
+switch at the end. Modify your electron version accordingly depending on the
+version that is installed on your system.

electron_app/src/electron-main.js

@@ -40,6 +40,16 @@ const { migrateFromOldOrigin } = require('./originMigrator');
 const windowStateKeeper = require('electron-window-state');
 const Store = require('electron-store');
 
+const fs = require('fs');
+const afs = fs.promises;
+
+let Seshat = null;
+
+try {
+    Seshat = require('matrix-seshat');
+} catch (e) {
+}
+
 if (argv["help"]) {
     console.log("Options:");
     console.log("  --profile-dir {path}: Path to where to store the profile.");
@@ -94,8 +104,11 @@ try {
     // Could not load local config, this is expected in most cases.
 }
 
+const eventStorePath = path.join(app.getPath('userData'), 'EventStore');
 const store = new Store({ name: "electron-config" });
 
+let eventIndex = null;
+
 let mainWindow = null;
 global.appQuitting = false;
 
@@ -225,6 +238,7 @@ ipcMain.on('ipcCall', async function(ev, payload) {
         case 'getConfig':
             ret = vectorConfig;
             break;
+
         default:
             mainWindow.webContents.send('ipcReply', {
                 id: payload.id,
@@ -239,6 +253,154 @@ ipcMain.on('ipcCall', async function(ev, payload) {
     });
 });
 
+ipcMain.on('seshat', async function(ev, payload) {
+    if (!mainWindow) return;
+
+    const sendError = (id, e) => {
+        const error = {
+            message: e.message
+        }
+
+        mainWindow.webContents.send('seshatReply', {
+            id:id,
+            error: error
+        });
+    }
+
+    const args = payload.args || [];
+    let ret;
+
+    switch (payload.name) {
+        case 'supportsEventIndexing':
+            if (Seshat === null) ret = false;
+            else ret = true;
+            break;
+
+        case 'initEventIndex':
+            if (eventIndex === null) {
+                try {
+                    await afs.mkdir(eventStorePath, {recursive: true});
+                    eventIndex = new Seshat(eventStorePath, {passphrase: "DEFAULT_PASSPHRASE"});
+                } catch (e) {
+                    sendError(payload.id, e);
+                    return;
+                }
+            }
+            break;
+
+        case 'closeEventIndex':
+            eventIndex = null;
+            break;
+
+        case 'deleteEventIndex':
+            const deleteFolderRecursive = async(p) =>  {
+                for (let entry of await afs.readdir(p)) {
+                    const curPath = path.join(p, entry);
+                    await afs.unlink(curPath);
+                }
+            }
+
+            try {
+                await deleteFolderRecursive(eventStorePath);
+            } catch (e) {
+            }
+
+            break;
+
+        case 'isEventIndexEmpty':
+            if (eventIndex === null) ret = true;
+            else ret = await eventIndex.isEmpty();
+            break;
+
+        case 'addEventToIndex':
+            try {
+                eventIndex.addEvent(args[0], args[1]);
+            } catch (e) {
+                sendError(payload.id, e);
+                return;
+            }
+            break;
+
+        case 'commitLiveEvents':
+            try {
+                ret = await eventIndex.commit();
+            } catch (e) {
+                sendError(payload.id, e);
+                return;
+            }
+            break;
+
+        case 'searchEventIndex':
+            try {
+                ret = await eventIndex.search(args[0]);
+            } catch (e) {
+                sendError(payload.id, e);
+                return;
+            }
+            break;
+
+        case 'addHistoricEvents':
+            if (eventIndex === null) ret = false;
+            else {
+                try {
+                    ret = await eventIndex.addHistoricEvents(
+                        args[0], args[1], args[2]);
+                } catch (e) {
+                    sendError(payload.id, e);
+                    return;
+                }
+            }
+            break;
+
+        case 'removeCrawlerCheckpoint':
+            if (eventIndex === null) ret = false;
+            else {
+                try {
+                    ret = await eventIndex.removeCrawlerCheckpoint(args[0]);
+                } catch (e) {
+                    sendError(payload.id, e);
+                    return;
+                }
+            }
+            break;
+
+        case 'addCrawlerCheckpoint':
+            if (eventIndex === null) ret = false;
+            else {
+                try {
+                    ret = await eventIndex.addCrawlerCheckpoint(args[0]);
+                } catch (e) {
+                    sendError(payload.id, e);
+                    return;
+                }
+            }
+            break;
+
+        case 'loadCheckpoints':
+            if (eventIndex === null) ret = [];
+            else {
+                try {
+                    ret = await eventIndex.loadCheckpoints();
+                } catch (e) {
+                    ret = [];
+                }
+            }
+            break;
+
+        default:
+            mainWindow.webContents.send('seshatReply', {
+                id: payload.id,
+                error: "Unknown IPC Call: " + payload.name,
+            });
+            return;
+    }
+
+    mainWindow.webContents.send('seshatReply', {
+        id: payload.id,
+        reply: ret,
+    });
+});
+
 app.commandLine.appendSwitch('--enable-usermedia-screen-capturing');
 
 const gotLock = app.requestSingleInstanceLock();

riot.im/develop/config.json

@@ -20,7 +20,8 @@
         "feature_many_integration_managers": "labs",
         "feature_mjolnir": "labs",
         "feature_dm_verification": "labs",
-        "feature_cross_signing": "labs"
+        "feature_cross_signing": "labs",
+        "feature_event_indexing": "labs"
     },
     "welcomeUserId": "@riot-bot:matrix.org",
     "piwik": {

src/vector/platform/ElectronPlatform.js

@@ -20,6 +20,7 @@ limitations under the License.
 */
 
 import VectorBasePlatform, {updateCheckStatusEnum} from './VectorBasePlatform';
+import BaseEventIndexManager from 'matrix-react-sdk/lib/indexing/BaseEventIndexManager';
 import dis from 'matrix-react-sdk/lib/dispatcher';
 import { _t } from 'matrix-react-sdk/lib/languageHandler';
 import rageshake from 'matrix-react-sdk/lib/rageshake/rageshake';
@@ -65,12 +66,104 @@ function getUpdateCheckStatus(status) {
     }
 }
 
+class SeshatIndexManager extends BaseEventIndexManager {
+    constructor() {
+        super();
+
+        this._pendingIpcCalls = {};
+        this._nextIpcCallId = 0;
+        ipcRenderer.on('seshatReply', this._onIpcReply.bind(this));
+    }
+
+    async _ipcCall(name: string, ...args: []): Promise<{}> {
+        // TODO this should be moved into the preload.js file.
+        const ipcCallId = ++this._nextIpcCallId;
+        return new Promise((resolve, reject) => {
+            this._pendingIpcCalls[ipcCallId] = {resolve, reject};
+            window.ipcRenderer.send('seshat', {id: ipcCallId, name, args});
+        });
+    }
+
+    _onIpcReply(ev: {}, payload: {}) {
+        if (payload.id === undefined) {
+            console.warn("Ignoring IPC reply with no ID");
+            return;
+        }
+
+        if (this._pendingIpcCalls[payload.id] === undefined) {
+            console.warn("Unknown IPC payload ID: " + payload.id);
+            return;
+        }
+
+        const callbacks = this._pendingIpcCalls[payload.id];
+        delete this._pendingIpcCalls[payload.id];
+        if (payload.error) {
+            callbacks.reject(payload.error);
+        } else {
+            callbacks.resolve(payload.reply);
+        }
+    }
+
+    async supportsEventIndexing(): Promise<boolean> {
+        return this._ipcCall('supportsEventIndexing');
+    }
+
+    async initEventIndex(): Promise<> {
+        return this._ipcCall('initEventIndex');
+    }
+
+    async addEventToIndex(ev: MatrixEvent, profile: MatrixProfile): Promise<> {
+        return this._ipcCall('addEventToIndex', ev, profile);
+    }
+
+    async isEventIndexEmpty(): Promise<boolean> {
+        return this._ipcCall('isEventIndexEmpty');
+    }
+
+    async commitLiveEvents(): Promise<> {
+        return this._ipcCall('commitLiveEvents');
+    }
+
+    async searchEventIndex(searchConfig: SearchConfig): Promise<SearchResult> {
+        return this._ipcCall('searchEventIndex', searchConfig);
+    }
+
+    async addHistoricEvents(
+        events: [HistoricEvent],
+        checkpoint: CrawlerCheckpoint | null,
+        oldCheckpoint: CrawlerCheckpoint | null,
+    ): Promise<> {
+        return this._ipcCall('addHistoricEvents', events, checkpoint, oldCheckpoint);
+    }
+
+    async addCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> {
+        return this._ipcCall('addCrawlerCheckpoint', checkpoint);
+    }
+
+    async removeCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> {
+        return this._ipcCall('removeCrawlerCheckpoint', checkpoint);
+    }
+
+    async loadCheckpoints(): Promise<[CrawlerCheckpoint]> {
+        return this._ipcCall('loadCheckpoints');
+    }
+
+    async closeEventIndex(): Promise<> {
+        return this._ipcCall('closeEventIndex');
+    }
+
+    async deleteEventIndex(): Promise<> {
+        return this._ipcCall('deleteEventIndex');
+    }
+}
+
 export default class ElectronPlatform extends VectorBasePlatform {
     constructor() {
         super();
 
         this._pendingIpcCalls = {};
         this._nextIpcCallId = 0;
+        this.eventIndexManager = new SeshatIndexManager();
 
         dis.register(_onAction);
         /*
@@ -294,4 +387,8 @@ export default class ElectronPlatform extends VectorBasePlatform {
             callbacks.resolve(payload.reply);
         }
     }
+
+    getEventIndexingManager(): BaseEventIndexManager | null {
+        return this.eventIndexManager;
+    }
 }