Merge pull request #2974 from vector-im/rav/deploy_script
Deploy script
This commit is contained in:
David Baker
GitHub
commit
6c4bf58ba5
|
|
@ -0,0 +1,183 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# download and unpack a riot-web tarball.
|
||||
#
|
||||
# Allows `bundles` to be extracted to a common directory, and a link to
|
||||
# config.json to be added.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import os.path
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
try:
|
||||
# python3
|
||||
from urllib.request import urlretrieve
|
||||
except ImportError:
|
||||
# python2
|
||||
from urllib import urlretrieve
|
||||
|
||||
class DeployException(Exception):
|
||||
pass
|
||||
|
||||
def create_relative_symlink(linkname, target):
|
||||
relpath = os.path.relpath(target, os.path.dirname(linkname))
|
||||
print ("Symlink %s -> %s" % (linkname, relpath))
|
||||
os.symlink(relpath, linkname)
|
||||
|
||||
|
||||
def move_bundles(source, dest):
|
||||
"""Move the contents of the 'bundles' directory to a common dir
|
||||
|
||||
We check that we will not be overwriting anything before we proceed.
|
||||
|
||||
Args:
|
||||
source (str): path to 'bundles' within the extracted tarball
|
||||
dest (str): target common directory
|
||||
"""
|
||||
|
||||
if not os.path.isdir(dest):
|
||||
os.mkdir(dest)
|
||||
|
||||
# build a map from source to destination, checking for non-existence as we go.
|
||||
renames = {}
|
||||
for f in os.listdir(source):
|
||||
dst = os.path.join(dest, f)
|
||||
if os.path.exists(dst):
|
||||
raise DeployException(
|
||||
"Not deploying. The bundle includes '%s' which we have previously deployed."
|
||||
% f
|
||||
)
|
||||
renames[os.path.join(source, f)] = dst
|
||||
|
||||
for (src, dst) in renames.iteritems():
|
||||
print ("Move %s -> %s" % (src, dst))
|
||||
os.rename(src, dst)
|
||||
|
||||
class Deployer:
|
||||
def __init__(self):
|
||||
self.packages_path = "."
|
||||
self.bundles_path = None
|
||||
self.should_clean = False
|
||||
self.config_location = None
|
||||
self.verify_signature = True
|
||||
|
||||
def deploy(self, tarball, extract_path):
|
||||
"""Download a tarball if necessary, and unpack it
|
||||
|
||||
Returns:
|
||||
(str) the path to the unpacked deployment
|
||||
"""
|
||||
print("Deploying %s to %s" % (tarball, extract_path))
|
||||
|
||||
name_str = os.path.basename(tarball).replace(".tar.gz", "")
|
||||
extracted_dir = os.path.join(extract_path, name_str)
|
||||
if os.path.exists(extracted_dir):
|
||||
raise DeployException('Cannot unpack %s: %s already exists' % (
|
||||
tarball, extracted_dir))
|
||||
|
||||
downloaded = False
|
||||
if tarball.startswith("http://") or tarball.startswith("https://"):
|
||||
tarball = self.download_and_verify(tarball)
|
||||
print("Downloaded file: %s" % tarball)
|
||||
downloaded = True
|
||||
|
||||
try:
|
||||
with tarfile.open(tarball) as tar:
|
||||
tar.extractall(extract_path)
|
||||
finally:
|
||||
if self.should_clean and downloaded:
|
||||
os.remove(tarball)
|
||||
|
||||
print ("Extracted into: %s" % extracted_dir)
|
||||
|
||||
if self.config_location:
|
||||
create_relative_symlink(
|
||||
target=self.config_location,
|
||||
linkname=os.path.join(extracted_dir, 'config.json')
|
||||
)
|
||||
|
||||
if self.bundles_path:
|
||||
extracted_bundles = os.path.join(extracted_dir, 'bundles')
|
||||
move_bundles(source=extracted_bundles, dest=self.bundles_path)
|
||||
|
||||
# replace the (hopefully now empty) extracted_bundles dir with a
|
||||
# symlink to the common dir.
|
||||
os.rmdir(extracted_bundles)
|
||||
create_relative_symlink(
|
||||
target=self.bundles_path,
|
||||
linkname=extracted_bundles,
|
||||
)
|
||||
return extracted_dir
|
||||
|
||||
def download_and_verify(self, url):
|
||||
tarball = self.download_file(url)
|
||||
|
||||
if self.verify_signature:
|
||||
sigfile = self.download_file(url + ".asc")
|
||||
subprocess.check_call(["gpg", "--verify", sigfile, tarball])
|
||||
|
||||
return tarball
|
||||
|
||||
def download_file(self, url):
|
||||
if not os.path.isdir(self.packages_path):
|
||||
os.mkdir(self.packages_path)
|
||||
local_filename = os.path.join(self.packages_path,
|
||||
url.split('/')[-1])
|
||||
sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
|
||||
sys.stdout.flush()
|
||||
urlretrieve(url, local_filename)
|
||||
print ("Done")
|
||||
return local_filename
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
|
||||
parser.add_argument(
|
||||
"-p", "--packages-dir", default="./packages", help=(
|
||||
"The directory to download the tarball into. (Default: '%(default)s')"
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e", "--extract-path", default="./deploys", help=(
|
||||
"The location to extract .tar.gz files to. (Default: '%(default)s')"
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"-b", "--bundles-dir", nargs='?', default="./bundles", help=(
|
||||
"A directory to move the contents of the 'bundles' directory to. A \
|
||||
symlink to the bundles directory will also be written inside the \
|
||||
extracted tarball. Example: './bundles'. \
|
||||
(Default: '%(default)s')"
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c", "--clean", action="store_true", default=False, help=(
|
||||
"Remove .tar.gz files after they have been downloaded and extracted. \
|
||||
(Default: %(default)s)"
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config", nargs='?', default='./config.json', help=(
|
||||
"Write a symlink at config.json in the extracted tarball to this \
|
||||
location. (Default: '%(default)s')"
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"tarball", help=(
|
||||
"filename of tarball, or URL to download."
|
||||
),
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
deployer = Deployer()
|
||||
deployer.packages_path = args.packages_dir
|
||||
deployer.bundles_path = args.bundles_dir
|
||||
deployer.should_clean = args.clean
|
||||
deployer.config_location = args.config
|
||||
|
||||
deployer.deploy(args.tarball, args.extract_path)
|
||||
|
|
@ -14,32 +14,17 @@ from __future__ import print_function
|
|||
import json, requests, tarfile, argparse, os, errno
|
||||
import time
|
||||
from urlparse import urljoin
|
||||
|
||||
from flask import Flask, jsonify, request, abort
|
||||
|
||||
from deploy import Deployer, DeployException
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
arg_jenkins_url = None
|
||||
deployer = None
|
||||
arg_extract_path = None
|
||||
arg_bundles_path = None
|
||||
arg_should_clean = None
|
||||
arg_symlink = None
|
||||
arg_config_location = None
|
||||
|
||||
class DeployException(Exception):
|
||||
pass
|
||||
|
||||
def download_file(url):
|
||||
local_filename = url.split('/')[-1]
|
||||
r = requests.get(url, stream=True)
|
||||
with open(local_filename, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=1024):
|
||||
if chunk: # filter out keep-alive new chunks
|
||||
f.write(chunk)
|
||||
return local_filename
|
||||
|
||||
def untar_to(tarball, dest):
|
||||
with tarfile.open(tarball) as tar:
|
||||
tar.extractall(dest)
|
||||
|
||||
def create_symlink(source, linkname):
|
||||
try:
|
||||
|
|
@ -137,17 +122,20 @@ def fetch_jenkins_build(job_name, build_num):
|
|||
# see half-written files.
|
||||
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
|
||||
try:
|
||||
deploy_tarball(tar_gz_url, build_dir)
|
||||
extracted_dir = deploy_tarball(tar_gz_url, build_dir)
|
||||
except DeployException as e:
|
||||
abort(400, e.message)
|
||||
|
||||
create_symlink(source=extracted_dir, linkname=arg_symlink)
|
||||
|
||||
return jsonify({})
|
||||
|
||||
def deploy_tarball(tar_gz_url, build_dir):
|
||||
"""Download a tarball from jenkins and deploy it as the new version
|
||||
"""
|
||||
print("Deploying %s to %s" % (tar_gz_url, build_dir))
|
||||
"""Download a tarball from jenkins and unpack it
|
||||
|
||||
Returns:
|
||||
(str) the path to the unpacked deployment
|
||||
"""
|
||||
if os.path.exists(build_dir):
|
||||
raise DeployException(
|
||||
"Not deploying. We have previously deployed this build."
|
||||
|
|
@ -156,62 +144,8 @@ def deploy_tarball(tar_gz_url, build_dir):
|
|||
|
||||
# we rely on the fact that flask only serves one request at a time to
|
||||
# ensure that we do not overwrite a tarball from a concurrent request.
|
||||
filename = download_file(tar_gz_url)
|
||||
print("Downloaded file: %s" % filename)
|
||||
|
||||
try:
|
||||
untar_to(filename, build_dir)
|
||||
print("Extracted to: %s" % build_dir)
|
||||
finally:
|
||||
if arg_should_clean:
|
||||
os.remove(filename)
|
||||
|
||||
name_str = filename.replace(".tar.gz", "")
|
||||
extracted_dir = os.path.join(build_dir, name_str)
|
||||
|
||||
if arg_config_location:
|
||||
create_symlink(source=arg_config_location, linkname=os.path.join(extracted_dir, 'config.json'))
|
||||
|
||||
if arg_bundles_path:
|
||||
extracted_bundles = os.path.join(extracted_dir, 'bundles')
|
||||
move_bundles(source=extracted_bundles, dest=arg_bundles_path)
|
||||
|
||||
# replace the (hopefully now empty) extracted_bundles dir with a
|
||||
# symlink to the common dir.
|
||||
relpath = os.path.relpath(arg_bundles_path, extracted_dir)
|
||||
os.rmdir(extracted_bundles)
|
||||
print ("Symlink %s -> %s" % (extracted_bundles, relpath))
|
||||
os.symlink(relpath, extracted_bundles)
|
||||
|
||||
create_symlink(source=extracted_dir, linkname=arg_symlink)
|
||||
|
||||
def move_bundles(source, dest):
|
||||
"""Move the contents of the 'bundles' directory to a common dir
|
||||
|
||||
We check that we will not be overwriting anything before we proceed.
|
||||
|
||||
Args:
|
||||
source (str): path to 'bundles' within the extracted tarball
|
||||
dest (str): target common directory
|
||||
"""
|
||||
|
||||
if not os.path.isdir(dest):
|
||||
os.mkdir(dest)
|
||||
|
||||
# build a map from source to destination, checking for non-existence as we go.
|
||||
renames = {}
|
||||
for f in os.listdir(source):
|
||||
dst = os.path.join(dest, f)
|
||||
if os.path.exists(dst):
|
||||
raise DeployException(
|
||||
"Not deploying. The bundle includes '%s' which we have previously deployed."
|
||||
% f
|
||||
)
|
||||
renames[os.path.join(source, f)] = dst
|
||||
|
||||
for (src, dst) in renames.iteritems():
|
||||
print ("Move %s -> %s" % (src, dst))
|
||||
os.rename(src, dst)
|
||||
return deployer.deploy(tar_gz_url, build_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -270,21 +204,34 @@ if __name__ == "__main__":
|
|||
else:
|
||||
arg_jenkins_url = args.jenkins + "/"
|
||||
arg_extract_path = args.extract
|
||||
arg_bundles_path = args.bundles_dir
|
||||
arg_should_clean = args.clean
|
||||
arg_symlink = args.symlink
|
||||
arg_config_location = args.config
|
||||
|
||||
if not os.path.isdir(arg_extract_path):
|
||||
os.mkdir(arg_extract_path)
|
||||
|
||||
deployer = Deployer()
|
||||
deployer.bundles_path = args.bundles_dir
|
||||
deployer.should_clean = args.clean
|
||||
deployer.config_location = args.config
|
||||
|
||||
# we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to
|
||||
# the jenkins server (and the jenkins server not being compromised and/or
|
||||
# github not serving it compromised source). If that's not good enough for
|
||||
# you, don't use riot.im/develop.
|
||||
deployer.verify_signature = False
|
||||
|
||||
if args.tarball_uri is not None:
|
||||
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
|
||||
deploy_tarball(args.tarball_uri, build_dir)
|
||||
else:
|
||||
print(
|
||||
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
|
||||
(args.port, arg_extract_path,
|
||||
" (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
|
||||
(args.port,
|
||||
arg_extract_path,
|
||||
" (clean after)" if deployer.should_clean else "",
|
||||
arg_symlink,
|
||||
arg_jenkins_url,
|
||||
deployer.config_location,
|
||||
)
|
||||
)
|
||||
app.run(host="0.0.0.0", port=args.port, debug=True)
|
||||
|
|
|
|||
Loading…
Reference in New Issue