247 lines
8.2 KiB
Python
Executable File
247 lines
8.2 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# auto-deploy script for https://riot.im/develop
|
|
#
|
|
# Listens for HTTP hits. When it gets one, downloads the artifact from jenkins
|
|
# and deploys it as the new version.
|
|
#
|
|
# Requires the following python packages:
|
|
#
|
|
# - requests
|
|
# - flask
|
|
#
|
|
from __future__ import print_function
|
|
import json, requests, tarfile, argparse, os, errno
|
|
import time
|
|
import traceback
|
|
from urlparse import urljoin
|
|
|
|
from flask import Flask, jsonify, request, abort
|
|
|
|
from deploy import Deployer, DeployException
|
|
|
|
app = Flask(__name__)
|
|
|
|
arg_jenkins_url = None
|
|
deployer = None
|
|
arg_extract_path = None
|
|
arg_symlink = None
|
|
|
|
def create_symlink(source, linkname):
|
|
try:
|
|
os.symlink(source, linkname)
|
|
except OSError, e:
|
|
if e.errno == errno.EEXIST:
|
|
# atomic modification
|
|
os.symlink(source, linkname + ".tmp")
|
|
os.rename(linkname + ".tmp", linkname)
|
|
else:
|
|
raise e
|
|
|
|
@app.route("/", methods=["POST"])
|
|
def on_receive_jenkins_poke():
|
|
# {
|
|
# "name": "VectorWebDevelop",
|
|
# "build": {
|
|
# "number": 8
|
|
# }
|
|
# }
|
|
incoming_json = request.get_json()
|
|
if not incoming_json:
|
|
abort(400, "No JSON provided!")
|
|
return
|
|
print("Incoming JSON: %s" % (incoming_json,))
|
|
|
|
job_name = incoming_json.get("name")
|
|
if not isinstance(job_name, basestring):
|
|
abort(400, "Bad job name: %s" % (job_name,))
|
|
return
|
|
|
|
build_num = incoming_json.get("build", {}).get("number", 0)
|
|
if not build_num or build_num <= 0 or not isinstance(build_num, int):
|
|
abort(400, "Missing or bad build number")
|
|
return
|
|
|
|
return fetch_jenkins_build(job_name, build_num)
|
|
|
|
def fetch_jenkins_build(job_name, build_num):
|
|
artifact_url = urljoin(
|
|
arg_jenkins_url, "job/%s/%s/api/json" % (job_name, build_num)
|
|
)
|
|
artifact_response = requests.get(artifact_url).json()
|
|
|
|
# {
|
|
# "actions": [],
|
|
# "artifacts": [
|
|
# {
|
|
# "displayPath": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz",
|
|
# "fileName": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz",
|
|
# "relativePath": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz"
|
|
# }
|
|
# ],
|
|
# "building": false,
|
|
# "description": null,
|
|
# "displayName": "#11",
|
|
# "duration": 137976,
|
|
# "estimatedDuration": 132008,
|
|
# "executor": null,
|
|
# "fullDisplayName": "VectorWebDevelop #11",
|
|
# "id": "11",
|
|
# "keepLog": false,
|
|
# "number": 11,
|
|
# "queueId": 12254,
|
|
# "result": "SUCCESS",
|
|
# "timestamp": 1454432640079,
|
|
# "url": "http://matrix.org/jenkins/job/VectorWebDevelop/11/",
|
|
# "builtOn": "",
|
|
# "changeSet": {},
|
|
# "culprits": []
|
|
# }
|
|
if artifact_response.get("result") != "SUCCESS":
|
|
abort(404, "Not deploying. Build was not marked as SUCCESS.")
|
|
return
|
|
|
|
if len(artifact_response.get("artifacts", [])) != 1:
|
|
abort(404, "Not deploying. Build has an unexpected number of artifacts.")
|
|
return
|
|
|
|
tar_gz_path = artifact_response["artifacts"][0]["relativePath"]
|
|
if not tar_gz_path.endswith(".tar.gz"):
|
|
abort(404, "Not deploying. Artifact is not a .tar.gz file")
|
|
return
|
|
|
|
tar_gz_url = urljoin(
|
|
arg_jenkins_url, "job/%s/%s/artifact/%s" % (job_name, build_num, tar_gz_path)
|
|
)
|
|
|
|
# we extract into a directory based on the build number. This avoids the
|
|
# problem of multiple builds building the same git version and thus having
|
|
# the same tarball name. That would lead to two potential problems:
|
|
# (a) sometimes jenkins serves corrupted artifacts; we would replace
|
|
# a good deploy with a bad one
|
|
# (b) we'll be overwriting the live deployment, which means people might
|
|
# see half-written files.
|
|
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
|
|
try:
|
|
extracted_dir = deploy_tarball(tar_gz_url, build_dir)
|
|
except DeployException as e:
|
|
traceback.print_exc()
|
|
abort(400, e.message)
|
|
|
|
create_symlink(source=extracted_dir, linkname=arg_symlink)
|
|
|
|
return jsonify({})
|
|
|
|
def deploy_tarball(tar_gz_url, build_dir):
|
|
"""Download a tarball from jenkins and unpack it
|
|
|
|
Returns:
|
|
(str) the path to the unpacked deployment
|
|
"""
|
|
if os.path.exists(build_dir):
|
|
raise DeployException(
|
|
"Not deploying. We have previously deployed this build."
|
|
)
|
|
os.mkdir(build_dir)
|
|
|
|
# we rely on the fact that flask only serves one request at a time to
|
|
# ensure that we do not overwrite a tarball from a concurrent request.
|
|
|
|
return deployer.deploy(tar_gz_url, build_dir)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
|
|
parser.add_argument(
|
|
"-j", "--jenkins", dest="jenkins", default="https://matrix.org/jenkins/", help=(
|
|
"The base URL of the Jenkins web server. This will be hit to get the\
|
|
built artifacts (the .gz file) for redeploying."
|
|
)
|
|
)
|
|
parser.add_argument(
|
|
"-p", "--port", dest="port", default=4000, type=int, help=(
|
|
"The port to listen on for requests from Jenkins."
|
|
)
|
|
)
|
|
parser.add_argument(
|
|
"-e", "--extract", dest="extract", default="./extracted", help=(
|
|
"The location to extract .tar.gz files to."
|
|
)
|
|
)
|
|
parser.add_argument(
|
|
"-b", "--bundles-dir", dest="bundles_dir", help=(
|
|
"A directory to move the contents of the 'bundles' directory to. A \
|
|
symlink to the bundles directory will also be written inside the \
|
|
extracted tarball. Example: './bundles'."
|
|
)
|
|
)
|
|
parser.add_argument(
|
|
"-c", "--clean", dest="clean", action="store_true", default=False, help=(
|
|
"Remove .tar.gz files after they have been downloaded and extracted."
|
|
)
|
|
)
|
|
parser.add_argument(
|
|
"-s", "--symlink", dest="symlink", default="./latest", help=(
|
|
"Write a symlink to this location pointing to the extracted tarball. \
|
|
New builds will keep overwriting this symlink. The symlink will point \
|
|
to the /vector directory INSIDE the tarball."
|
|
)
|
|
)
|
|
|
|
def _raise(ex):
|
|
raise ex
|
|
|
|
# --config config.json=../../config.json --config config.localhost.json=./localhost.json
|
|
parser.add_argument(
|
|
"--config", action="append", dest="configs",
|
|
type=lambda kv: kv.split("=", 1) if "=" in kv else _raise(Exception("Missing =")), help=(
|
|
"A list of configs to symlink into the extracted tarball. \
|
|
For example, --config config.json=../config.json config2.json=../test/config.json"
|
|
)
|
|
)
|
|
parser.add_argument(
|
|
"--test", dest="tarball_uri", help=(
|
|
"Don't start an HTTP listener. Instead download a build from Jenkins \
|
|
immediately."
|
|
),
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
if args.jenkins.endswith("/"): # important for urljoin
|
|
arg_jenkins_url = args.jenkins
|
|
else:
|
|
arg_jenkins_url = args.jenkins + "/"
|
|
arg_extract_path = args.extract
|
|
arg_symlink = args.symlink
|
|
|
|
if not os.path.isdir(arg_extract_path):
|
|
os.mkdir(arg_extract_path)
|
|
|
|
deployer = Deployer()
|
|
deployer.bundles_path = args.bundles_dir
|
|
deployer.should_clean = args.clean
|
|
deployer.config_locations = dict(args.configs) if args.configs else {}
|
|
|
|
|
|
# we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to
|
|
# the jenkins server (and the jenkins server not being compromised and/or
|
|
# github not serving it compromised source). If that's not good enough for
|
|
# you, don't use riot.im/develop.
|
|
deployer.verify_signature = False
|
|
|
|
if args.tarball_uri is not None:
|
|
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
|
|
deploy_tarball(args.tarball_uri, build_dir)
|
|
else:
|
|
print(
|
|
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config locations: %s" %
|
|
(args.port,
|
|
arg_extract_path,
|
|
" (clean after)" if deployer.should_clean else "",
|
|
arg_symlink,
|
|
arg_jenkins_url,
|
|
deployer.config_locations,
|
|
)
|
|
)
|
|
app.run(host="0.0.0.0", port=args.port, debug=True)
|