Merge pull request #2974 from vector-im/rav/deploy_script

Deploy script

scripts/deploy.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python
+#
+# download and unpack a riot-web tarball.
+#
+# Allows `bundles` to be extracted to a common directory, and a link to
+# config.json to be added.
+
+from __future__ import print_function
+
+import argparse
+import os
+import os.path
+import subprocess
+import sys
+import tarfile
+
+try:
+    # python3
+    from urllib.request import urlretrieve
+except ImportError:
+    # python2
+    from urllib import urlretrieve
+
+class DeployException(Exception):
+    pass
+
+def create_relative_symlink(linkname, target):
+    relpath = os.path.relpath(target, os.path.dirname(linkname))
+    print ("Symlink %s -> %s" % (linkname, relpath))
+    os.symlink(relpath, linkname)
+
+
+def move_bundles(source, dest):
+    """Move the contents of the 'bundles' directory to a common dir
+
+    We check that we will not be overwriting anything before we proceed.
+
+    Args:
+        source (str): path to 'bundles' within the extracted tarball
+        dest (str): target common directory
+    """
+
+    if not os.path.isdir(dest):
+        os.mkdir(dest)
+
+    # build a map from source to destination, checking for non-existence as we go.
+    renames = {}
+    for f in os.listdir(source):
+        dst = os.path.join(dest, f)
+        if os.path.exists(dst):
+            raise DeployException(
+                "Not deploying. The bundle includes '%s' which we have previously deployed."
+                % f
+            )
+        renames[os.path.join(source, f)] = dst
+
+    for (src, dst) in renames.iteritems():
+        print ("Move %s -> %s" % (src, dst))
+        os.rename(src, dst)
+
+class Deployer:
+    def __init__(self):
+        self.packages_path = "."
+        self.bundles_path = None
+        self.should_clean = False
+        self.config_location = None
+        self.verify_signature = True
+
+    def deploy(self, tarball, extract_path):
+        """Download a tarball if necessary, and unpack it
+
+        Returns:
+            (str) the path to the unpacked deployment
+        """
+        print("Deploying %s to %s" % (tarball, extract_path))
+
+        name_str = os.path.basename(tarball).replace(".tar.gz", "")
+        extracted_dir = os.path.join(extract_path, name_str)
+        if os.path.exists(extracted_dir):
+            raise DeployException('Cannot unpack %s: %s already exists' % (
+                tarball, extracted_dir))
+
+        downloaded = False
+        if tarball.startswith("http://") or tarball.startswith("https://"):
+            tarball = self.download_and_verify(tarball)
+            print("Downloaded file: %s" % tarball)
+            downloaded = True
+
+        try:
+            with tarfile.open(tarball) as tar:
+                tar.extractall(extract_path)
+        finally:
+            if self.should_clean and downloaded:
+                os.remove(tarball)
+
+        print ("Extracted into: %s" % extracted_dir)
+
+        if self.config_location:
+            create_relative_symlink(
+                target=self.config_location,
+                linkname=os.path.join(extracted_dir, 'config.json')
+            )
+
+        if self.bundles_path:
+            extracted_bundles = os.path.join(extracted_dir, 'bundles')
+            move_bundles(source=extracted_bundles, dest=self.bundles_path)
+
+            # replace the (hopefully now empty) extracted_bundles dir with a
+            # symlink to the common dir.
+            os.rmdir(extracted_bundles)
+            create_relative_symlink(
+                target=self.bundles_path,
+                linkname=extracted_bundles,
+            )
+        return extracted_dir
+
+    def download_and_verify(self, url):
+        tarball = self.download_file(url)
+
+        if self.verify_signature:
+            sigfile = self.download_file(url + ".asc")
+            subprocess.check_call(["gpg", "--verify", sigfile, tarball])
+
+        return tarball
+
+    def download_file(self, url):
+        if not os.path.isdir(self.packages_path):
+            os.mkdir(self.packages_path)
+        local_filename = os.path.join(self.packages_path,
+                                      url.split('/')[-1])
+        sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
+        sys.stdout.flush()
+        urlretrieve(url, local_filename)
+        print ("Done")
+        return local_filename
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
+    parser.add_argument(
+        "-p", "--packages-dir", default="./packages", help=(
+            "The directory to download the tarball into. (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "-e", "--extract-path", default="./deploys", help=(
+            "The location to extract .tar.gz files to. (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "-b", "--bundles-dir", nargs='?', default="./bundles", help=(
+            "A directory to move the contents of the 'bundles' directory to. A \
+            symlink to the bundles directory will also be written inside the \
+            extracted tarball. Example: './bundles'. \
+            (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "-c", "--clean", action="store_true", default=False, help=(
+            "Remove .tar.gz files after they have been downloaded and extracted. \
+            (Default: %(default)s)"
+        )
+    )
+    parser.add_argument(
+        "--config", nargs='?', default='./config.json', help=(
+            "Write a symlink at config.json in the extracted tarball to this \
+            location. (Default: '%(default)s')"
+        )
+    )
+    parser.add_argument(
+        "tarball", help=(
+            "filename of tarball, or URL to download."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    deployer = Deployer()
+    deployer.packages_path = args.packages_dir
+    deployer.bundles_path = args.bundles_dir
+    deployer.should_clean = args.clean
+    deployer.config_location = args.config
+
+    deployer.deploy(args.tarball, args.extract_path)

scripts/redeploy.py

@@ -14,32 +14,17 @@ from __future__ import print_function
 import json, requests, tarfile, argparse, os, errno
 import time
 from urlparse import urljoin
+
 from flask import Flask, jsonify, request, abort
 
+from deploy import Deployer, DeployException
+
 app = Flask(__name__)
 
 arg_jenkins_url = None
+deployer = None
 arg_extract_path = None
-arg_bundles_path = None
-arg_should_clean = None
 arg_symlink = None
-arg_config_location = None
-
-class DeployException(Exception):
-    pass
-
-def download_file(url):
-    local_filename = url.split('/')[-1]
-    r = requests.get(url, stream=True)
-    with open(local_filename, 'wb') as f:
-        for chunk in r.iter_content(chunk_size=1024):
-            if chunk: # filter out keep-alive new chunks
-                f.write(chunk)
-    return local_filename
-
-def untar_to(tarball, dest):
-    with tarfile.open(tarball) as tar:
-        tar.extractall(dest)
 
 def create_symlink(source, linkname):
     try:
@@ -137,17 +122,20 @@ def fetch_jenkins_build(job_name, build_num):
     #       see half-written files.
     build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
     try:
-        deploy_tarball(tar_gz_url, build_dir)
+        extracted_dir = deploy_tarball(tar_gz_url, build_dir)
     except DeployException as e:
         abort(400, e.message)
 
+    create_symlink(source=extracted_dir, linkname=arg_symlink)
+
     return jsonify({})
 
 def deploy_tarball(tar_gz_url, build_dir):
-    """Download a tarball from jenkins and deploy it as the new version
+    """Download a tarball from jenkins and unpack it
-    """
-    print("Deploying %s to %s" % (tar_gz_url, build_dir))
 
+    Returns:
+        (str) the path to the unpacked deployment
+    """
     if os.path.exists(build_dir):
         raise DeployException(
             "Not deploying. We have previously deployed this build."
@@ -156,62 +144,8 @@ def deploy_tarball(tar_gz_url, build_dir):
 
     # we rely on the fact that flask only serves one request at a time to
     # ensure that we do not overwrite a tarball from a concurrent request.
-    filename = download_file(tar_gz_url)
-    print("Downloaded file: %s" % filename)
 
-    try:
+    return deployer.deploy(tar_gz_url, build_dir)
-        untar_to(filename, build_dir)
-        print("Extracted to: %s" % build_dir)
-    finally:
-        if arg_should_clean:
-            os.remove(filename)
-
-    name_str = filename.replace(".tar.gz", "")
-    extracted_dir = os.path.join(build_dir, name_str)
-
-    if arg_config_location:
-        create_symlink(source=arg_config_location, linkname=os.path.join(extracted_dir, 'config.json'))
-
-    if arg_bundles_path:
-        extracted_bundles = os.path.join(extracted_dir, 'bundles')
-        move_bundles(source=extracted_bundles, dest=arg_bundles_path)
-
-        # replace the (hopefully now empty) extracted_bundles dir with a
-        # symlink to the common dir.
-        relpath = os.path.relpath(arg_bundles_path, extracted_dir)
-        os.rmdir(extracted_bundles)
-        print ("Symlink %s -> %s" % (extracted_bundles, relpath))
-        os.symlink(relpath, extracted_bundles)
-
-    create_symlink(source=extracted_dir, linkname=arg_symlink)
-
-def move_bundles(source, dest):
-    """Move the contents of the 'bundles' directory to a common dir
-
-    We check that we will not be overwriting anything before we proceed.
-
-    Args:
-        source (str): path to 'bundles' within the extracted tarball
-        dest (str): target common directory
-    """
-
-    if not os.path.isdir(dest):
-        os.mkdir(dest)
-
-    # build a map from source to destination, checking for non-existence as we go.
-    renames = {}
-    for f in os.listdir(source):
-        dst = os.path.join(dest, f)
-        if os.path.exists(dst):
-            raise DeployException(
-                "Not deploying. The bundle includes '%s' which we have previously deployed."
-                % f
-            )
-        renames[os.path.join(source, f)] = dst
-
-    for (src, dst) in renames.iteritems():
-        print ("Move %s -> %s" % (src, dst))
-        os.rename(src, dst)
 
 
 if __name__ == "__main__":
@@ -270,21 +204,34 @@ if __name__ == "__main__":
     else:
         arg_jenkins_url = args.jenkins + "/"
     arg_extract_path = args.extract
-    arg_bundles_path = args.bundles_dir
-    arg_should_clean = args.clean
     arg_symlink = args.symlink
-    arg_config_location = args.config
 
     if not os.path.isdir(arg_extract_path):
         os.mkdir(arg_extract_path)
 
+    deployer = Deployer()
+    deployer.bundles_path = args.bundles_dir
+    deployer.should_clean = args.clean
+    deployer.config_location = args.config
+
+    # we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to
+    # the jenkins server (and the jenkins server not being compromised and/or
+    # github not serving it compromised source). If that's not good enough for
+    # you, don't use riot.im/develop.
+    deployer.verify_signature = False
+
     if args.tarball_uri is not None:
         build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
         deploy_tarball(args.tarball_uri, build_dir)
     else:
         print(
             "Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
-            (args.port, arg_extract_path,
+            (args.port,
-             " (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
+             arg_extract_path,
+             " (clean after)" if deployer.should_clean else "",
+             arg_symlink,
+             arg_jenkins_url,
+             deployer.config_location,
+            )
         )
         app.run(host="0.0.0.0", port=args.port, debug=True)