diff --git a/scripts/deploy.py b/scripts/deploy.py new file mode 100755 index 00000000..c96b46e8 --- /dev/null +++ b/scripts/deploy.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# +# download and unpack a riot-web tarball. +# +# Allows `bundles` to be extracted to a common directory, and a link to +# config.json to be added. + +from __future__ import print_function + +import argparse +import os +import os.path +import subprocess +import sys +import tarfile + +try: + # python3 + from urllib.request import urlretrieve +except ImportError: + # python2 + from urllib import urlretrieve + +class DeployException(Exception): + pass + +def create_relative_symlink(linkname, target): + relpath = os.path.relpath(target, os.path.dirname(linkname)) + print ("Symlink %s -> %s" % (linkname, relpath)) + os.symlink(relpath, linkname) + + +def move_bundles(source, dest): + """Move the contents of the 'bundles' directory to a common dir + + We check that we will not be overwriting anything before we proceed. + + Args: + source (str): path to 'bundles' within the extracted tarball + dest (str): target common directory + """ + + if not os.path.isdir(dest): + os.mkdir(dest) + + # build a map from source to destination, checking for non-existence as we go. + renames = {} + for f in os.listdir(source): + dst = os.path.join(dest, f) + if os.path.exists(dst): + raise DeployException( + "Not deploying. The bundle includes '%s' which we have previously deployed." + % f + ) + renames[os.path.join(source, f)] = dst + + for (src, dst) in renames.iteritems(): + print ("Move %s -> %s" % (src, dst)) + os.rename(src, dst) + +class Deployer: + def __init__(self): + self.packages_path = "." + self.bundles_path = None + self.should_clean = False + self.config_location = None + self.verify_signature = True + + def deploy(self, tarball, extract_path): + """Download a tarball if necessary, and unpack it + + Returns: + (str) the path to the unpacked deployment + """ + print("Deploying %s to %s" % (tarball, extract_path)) + + name_str = os.path.basename(tarball).replace(".tar.gz", "") + extracted_dir = os.path.join(extract_path, name_str) + if os.path.exists(extracted_dir): + raise DeployException('Cannot unpack %s: %s already exists' % ( + tarball, extracted_dir)) + + downloaded = False + if tarball.startswith("http://") or tarball.startswith("https://"): + tarball = self.download_and_verify(tarball) + print("Downloaded file: %s" % tarball) + downloaded = True + + try: + with tarfile.open(tarball) as tar: + tar.extractall(extract_path) + finally: + if self.should_clean and downloaded: + os.remove(tarball) + + print ("Extracted into: %s" % extracted_dir) + + if self.config_location: + create_relative_symlink( + target=self.config_location, + linkname=os.path.join(extracted_dir, 'config.json') + ) + + if self.bundles_path: + extracted_bundles = os.path.join(extracted_dir, 'bundles') + move_bundles(source=extracted_bundles, dest=self.bundles_path) + + # replace the (hopefully now empty) extracted_bundles dir with a + # symlink to the common dir. + os.rmdir(extracted_bundles) + create_relative_symlink( + target=self.bundles_path, + linkname=extracted_bundles, + ) + return extracted_dir + + def download_and_verify(self, url): + tarball = self.download_file(url) + + if self.verify_signature: + sigfile = self.download_file(url + ".asc") + subprocess.check_call(["gpg", "--verify", sigfile, tarball]) + + return tarball + + def download_file(self, url): + if not os.path.isdir(self.packages_path): + os.mkdir(self.packages_path) + local_filename = os.path.join(self.packages_path, + url.split('/')[-1]) + sys.stdout.write("Downloading %s -> %s..." % (url, local_filename)) + sys.stdout.flush() + urlretrieve(url, local_filename) + print ("Done") + return local_filename + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Deploy a Riot build on a web server.") + parser.add_argument( + "-p", "--packages-dir", default="./packages", help=( + "The directory to download the tarball into. (Default: '%(default)s')" + ) + ) + parser.add_argument( + "-e", "--extract-path", default="./deploys", help=( + "The location to extract .tar.gz files to. (Default: '%(default)s')" + ) + ) + parser.add_argument( + "-b", "--bundles-dir", nargs='?', default="./bundles", help=( + "A directory to move the contents of the 'bundles' directory to. A \ + symlink to the bundles directory will also be written inside the \ + extracted tarball. Example: './bundles'. \ + (Default: '%(default)s')" + ) + ) + parser.add_argument( + "-c", "--clean", action="store_true", default=False, help=( + "Remove .tar.gz files after they have been downloaded and extracted. \ + (Default: %(default)s)" + ) + ) + parser.add_argument( + "--config", nargs='?', default='./config.json', help=( + "Write a symlink at config.json in the extracted tarball to this \ + location. (Default: '%(default)s')" + ) + ) + parser.add_argument( + "tarball", help=( + "filename of tarball, or URL to download." + ), + ) + + args = parser.parse_args() + + deployer = Deployer() + deployer.packages_path = args.packages_dir + deployer.bundles_path = args.bundles_dir + deployer.should_clean = args.clean + deployer.config_location = args.config + + deployer.deploy(args.tarball, args.extract_path) diff --git a/scripts/redeploy.py b/scripts/redeploy.py index 4394f292..598f6c52 100755 --- a/scripts/redeploy.py +++ b/scripts/redeploy.py @@ -14,32 +14,17 @@ from __future__ import print_function import json, requests, tarfile, argparse, os, errno import time from urlparse import urljoin + from flask import Flask, jsonify, request, abort +from deploy import Deployer, DeployException + app = Flask(__name__) arg_jenkins_url = None +deployer = None arg_extract_path = None -arg_bundles_path = None -arg_should_clean = None arg_symlink = None -arg_config_location = None - -class DeployException(Exception): - pass - -def download_file(url): - local_filename = url.split('/')[-1] - r = requests.get(url, stream=True) - with open(local_filename, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - return local_filename - -def untar_to(tarball, dest): - with tarfile.open(tarball) as tar: - tar.extractall(dest) def create_symlink(source, linkname): try: @@ -137,17 +122,20 @@ def fetch_jenkins_build(job_name, build_num): # see half-written files. build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num)) try: - deploy_tarball(tar_gz_url, build_dir) + extracted_dir = deploy_tarball(tar_gz_url, build_dir) except DeployException as e: abort(400, e.message) + create_symlink(source=extracted_dir, linkname=arg_symlink) + return jsonify({}) def deploy_tarball(tar_gz_url, build_dir): - """Download a tarball from jenkins and deploy it as the new version - """ - print("Deploying %s to %s" % (tar_gz_url, build_dir)) + """Download a tarball from jenkins and unpack it + Returns: + (str) the path to the unpacked deployment + """ if os.path.exists(build_dir): raise DeployException( "Not deploying. We have previously deployed this build." @@ -156,62 +144,8 @@ def deploy_tarball(tar_gz_url, build_dir): # we rely on the fact that flask only serves one request at a time to # ensure that we do not overwrite a tarball from a concurrent request. - filename = download_file(tar_gz_url) - print("Downloaded file: %s" % filename) - try: - untar_to(filename, build_dir) - print("Extracted to: %s" % build_dir) - finally: - if arg_should_clean: - os.remove(filename) - - name_str = filename.replace(".tar.gz", "") - extracted_dir = os.path.join(build_dir, name_str) - - if arg_config_location: - create_symlink(source=arg_config_location, linkname=os.path.join(extracted_dir, 'config.json')) - - if arg_bundles_path: - extracted_bundles = os.path.join(extracted_dir, 'bundles') - move_bundles(source=extracted_bundles, dest=arg_bundles_path) - - # replace the (hopefully now empty) extracted_bundles dir with a - # symlink to the common dir. - relpath = os.path.relpath(arg_bundles_path, extracted_dir) - os.rmdir(extracted_bundles) - print ("Symlink %s -> %s" % (extracted_bundles, relpath)) - os.symlink(relpath, extracted_bundles) - - create_symlink(source=extracted_dir, linkname=arg_symlink) - -def move_bundles(source, dest): - """Move the contents of the 'bundles' directory to a common dir - - We check that we will not be overwriting anything before we proceed. - - Args: - source (str): path to 'bundles' within the extracted tarball - dest (str): target common directory - """ - - if not os.path.isdir(dest): - os.mkdir(dest) - - # build a map from source to destination, checking for non-existence as we go. - renames = {} - for f in os.listdir(source): - dst = os.path.join(dest, f) - if os.path.exists(dst): - raise DeployException( - "Not deploying. The bundle includes '%s' which we have previously deployed." - % f - ) - renames[os.path.join(source, f)] = dst - - for (src, dst) in renames.iteritems(): - print ("Move %s -> %s" % (src, dst)) - os.rename(src, dst) + return deployer.deploy(tar_gz_url, build_dir) if __name__ == "__main__": @@ -270,21 +204,34 @@ if __name__ == "__main__": else: arg_jenkins_url = args.jenkins + "/" arg_extract_path = args.extract - arg_bundles_path = args.bundles_dir - arg_should_clean = args.clean arg_symlink = args.symlink - arg_config_location = args.config if not os.path.isdir(arg_extract_path): os.mkdir(arg_extract_path) + deployer = Deployer() + deployer.bundles_path = args.bundles_dir + deployer.should_clean = args.clean + deployer.config_location = args.config + + # we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to + # the jenkins server (and the jenkins server not being compromised and/or + # github not serving it compromised source). If that's not good enough for + # you, don't use riot.im/develop. + deployer.verify_signature = False + if args.tarball_uri is not None: build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time())) deploy_tarball(args.tarball_uri, build_dir) else: print( "Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" % - (args.port, arg_extract_path, - " (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location) + (args.port, + arg_extract_path, + " (clean after)" if deployer.should_clean else "", + arg_symlink, + arg_jenkins_url, + deployer.config_location, + ) ) app.run(host="0.0.0.0", port=args.port, debug=True)