forked from matrix/element-web
Convert redeploy.py to buildkite
This commit is contained in:
parent
6c4c908678
commit
af6ad9355d
|
@ -2,7 +2,7 @@
|
||||||
#
|
#
|
||||||
# auto-deploy script for https://riot.im/develop
|
# auto-deploy script for https://riot.im/develop
|
||||||
#
|
#
|
||||||
# Listens for HTTP hits. When it gets one, downloads the artifact from jenkins
|
# Listens for HTTP hits. When it gets one, downloads the artifact from buildkite
|
||||||
# and deploys it as the new version.
|
# and deploys it as the new version.
|
||||||
#
|
#
|
||||||
# Requires the following python packages:
|
# Requires the following python packages:
|
||||||
|
@ -16,6 +16,8 @@ import time
|
||||||
import traceback
|
import traceback
|
||||||
from urlparse import urljoin
|
from urlparse import urljoin
|
||||||
import glob
|
import glob
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
|
||||||
from flask import Flask, jsonify, request, abort
|
from flask import Flask, jsonify, request, abort
|
||||||
|
|
||||||
|
@ -23,10 +25,11 @@ from deploy import Deployer, DeployException
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
|
||||||
arg_jenkins_url = None
|
|
||||||
deployer = None
|
deployer = None
|
||||||
arg_extract_path = None
|
arg_extract_path = None
|
||||||
arg_symlink = None
|
arg_symlink = None
|
||||||
|
arg_webbook_token = None
|
||||||
|
arg_api_token = None
|
||||||
|
|
||||||
def create_symlink(source, linkname):
|
def create_symlink(source, linkname):
|
||||||
try:
|
try:
|
||||||
|
@ -39,81 +42,98 @@ def create_symlink(source, linkname):
|
||||||
else:
|
else:
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
def req_headers():
|
||||||
|
return {
|
||||||
|
"Authorization": "Bearer %s" % (arg_api_token,),
|
||||||
|
}
|
||||||
|
|
||||||
@app.route("/", methods=["POST"])
|
@app.route("/", methods=["POST"])
|
||||||
def on_receive_jenkins_poke():
|
def on_receive_buildkite_poke():
|
||||||
# {
|
got_webhook_token = request.headers.get('X-Buildkite-Token')
|
||||||
# "name": "VectorWebDevelop",
|
if got_webhook_token != arg_webbook_token:
|
||||||
# "build": {
|
print("Denying request with incorrect webhook token: %s" % (got_webhook_token,))
|
||||||
# "number": 8
|
abort(400, "Incorrect webhook token")
|
||||||
# }
|
return
|
||||||
# }
|
|
||||||
|
required_api_prefix = None
|
||||||
|
if arg_buildkit_org is not None:
|
||||||
|
required_api_prefix = 'https://api.buildkite.com/v2/organizations/%s' % (arg_buildkit_org,)
|
||||||
|
|
||||||
incoming_json = request.get_json()
|
incoming_json = request.get_json()
|
||||||
if not incoming_json:
|
if not incoming_json:
|
||||||
abort(400, "No JSON provided!")
|
abort(400, "No JSON provided!")
|
||||||
return
|
return
|
||||||
print("Incoming JSON: %s" % (incoming_json,))
|
print("Incoming JSON: %s" % (incoming_json,))
|
||||||
|
|
||||||
job_name = incoming_json.get("name")
|
event = incoming_json.get("event")
|
||||||
if not isinstance(job_name, basestring):
|
if event is None:
|
||||||
abort(400, "Bad job name: %s" % (job_name,))
|
abort(400, "No 'event' specified")
|
||||||
return
|
return
|
||||||
|
|
||||||
build_num = incoming_json.get("build", {}).get("number", 0)
|
if event == 'ping':
|
||||||
if not build_num or build_num <= 0 or not isinstance(build_num, int):
|
print("Got ping request - responding")
|
||||||
abort(400, "Missing or bad build number")
|
return jsonify({'response': 'pong!'})
|
||||||
|
|
||||||
|
if event != 'build.finished':
|
||||||
|
print("Rejecting '%s' event")
|
||||||
|
abort(400, "Unrecognised event")
|
||||||
return
|
return
|
||||||
|
|
||||||
return fetch_jenkins_build(job_name, build_num)
|
build_obj = incoming_json.get("build")
|
||||||
|
if build_obj is None:
|
||||||
def fetch_jenkins_build(job_name, build_num):
|
abort(400, "No 'build' object")
|
||||||
artifact_url = urljoin(
|
|
||||||
arg_jenkins_url, "job/%s/%s/api/json" % (job_name, build_num)
|
|
||||||
)
|
|
||||||
artifact_response = requests.get(artifact_url).json()
|
|
||||||
|
|
||||||
# {
|
|
||||||
# "actions": [],
|
|
||||||
# "artifacts": [
|
|
||||||
# {
|
|
||||||
# "displayPath": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz",
|
|
||||||
# "fileName": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz",
|
|
||||||
# "relativePath": "vector-043f6991a4ed-react-20f77d1224ef-js-0a7efe3e8bd5.tar.gz"
|
|
||||||
# }
|
|
||||||
# ],
|
|
||||||
# "building": false,
|
|
||||||
# "description": null,
|
|
||||||
# "displayName": "#11",
|
|
||||||
# "duration": 137976,
|
|
||||||
# "estimatedDuration": 132008,
|
|
||||||
# "executor": null,
|
|
||||||
# "fullDisplayName": "VectorWebDevelop #11",
|
|
||||||
# "id": "11",
|
|
||||||
# "keepLog": false,
|
|
||||||
# "number": 11,
|
|
||||||
# "queueId": 12254,
|
|
||||||
# "result": "SUCCESS",
|
|
||||||
# "timestamp": 1454432640079,
|
|
||||||
# "url": "http://matrix.org/jenkins/job/VectorWebDevelop/11/",
|
|
||||||
# "builtOn": "",
|
|
||||||
# "changeSet": {},
|
|
||||||
# "culprits": []
|
|
||||||
# }
|
|
||||||
if artifact_response.get("result") != "SUCCESS":
|
|
||||||
abort(404, "Not deploying. Build was not marked as SUCCESS.")
|
|
||||||
return
|
return
|
||||||
|
|
||||||
if len(artifact_response.get("artifacts", [])) != 1:
|
build_url = build_obj.get('url')
|
||||||
abort(404, "Not deploying. Build has an unexpected number of artifacts.")
|
if build_url is None:
|
||||||
|
abort(400, "build has no url")
|
||||||
return
|
return
|
||||||
|
|
||||||
tar_gz_path = artifact_response["artifacts"][0]["relativePath"]
|
if required_api_prefix is not None and not build_url.startswith(required_api_prefix):
|
||||||
if not tar_gz_path.endswith(".tar.gz"):
|
print("Denying poke for build url with incorrect prefix: %s" % (build_url,))
|
||||||
abort(404, "Not deploying. Artifact is not a .tar.gz file")
|
abort(400, "Invalid build url")
|
||||||
return
|
return
|
||||||
|
|
||||||
tar_gz_url = urljoin(
|
build_num = build_obj.get('number')
|
||||||
arg_jenkins_url, "job/%s/%s/artifact/%s" % (job_name, build_num, tar_gz_path)
|
if build_num is None:
|
||||||
)
|
abort(400, "build has no number")
|
||||||
|
return
|
||||||
|
|
||||||
|
pipeline_obj = incoming_json.get("pipeline")
|
||||||
|
if pipeline_obj is None:
|
||||||
|
abort(400, "No 'pipeline' object")
|
||||||
|
return
|
||||||
|
|
||||||
|
pipeline_name = pipeline_obj.get('name')
|
||||||
|
if pipeline_name is None:
|
||||||
|
abort(400, "pipeline has no name")
|
||||||
|
return
|
||||||
|
|
||||||
|
artifacts_url = build_url + "/artifacts"
|
||||||
|
artifacts_resp = requests.get(artifacts_url, headers=req_headers())
|
||||||
|
artifacts_resp.raise_for_status()
|
||||||
|
artifacts_array = artifacts_resp.json()
|
||||||
|
|
||||||
|
for artifact in artifacts_array:
|
||||||
|
artifact_to_deploy = None
|
||||||
|
if re.match(r"dist/.*.tar.gz", artifact['path']):
|
||||||
|
artifact_to_deploy = artifact
|
||||||
|
if artifact_to_deploy is None:
|
||||||
|
print("No suitable artifacts found")
|
||||||
|
return jsonify({})
|
||||||
|
|
||||||
|
# double paranoia check: make sure the artifact is on the right org too
|
||||||
|
if required_api_prefix is not None and not artifact_to_deploy['url'].startswith(required_api_prefix):
|
||||||
|
print("Denying poke for build url with incorrect prefix: %s" % (artifact_to_deploy['url'],))
|
||||||
|
abort(400, "Refusing to deploy artifact from URL %s", artifact_to_deploy['url'])
|
||||||
|
return
|
||||||
|
|
||||||
|
return deploy_buildkite_artifact(artifact_to_deploy, pipeline_name, build_num)
|
||||||
|
|
||||||
|
def deploy_buildkite_artifact(artifact, pipeline_name, build_num):
|
||||||
|
artifact_response = requests.get(artifact['url'], headers=req_headers())
|
||||||
|
artifact_response.raise_for_status()
|
||||||
|
artifact_obj = artifact_response.json()
|
||||||
|
|
||||||
# we extract into a directory based on the build number. This avoids the
|
# we extract into a directory based on the build number. This avoids the
|
||||||
# problem of multiple builds building the same git version and thus having
|
# problem of multiple builds building the same git version and thus having
|
||||||
|
@ -122,9 +142,9 @@ def fetch_jenkins_build(job_name, build_num):
|
||||||
# a good deploy with a bad one
|
# a good deploy with a bad one
|
||||||
# (b) we'll be overwriting the live deployment, which means people might
|
# (b) we'll be overwriting the live deployment, which means people might
|
||||||
# see half-written files.
|
# see half-written files.
|
||||||
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
|
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (pipeline_name, build_num))
|
||||||
try:
|
try:
|
||||||
extracted_dir = deploy_tarball(tar_gz_url, build_dir)
|
extracted_dir = deploy_tarball(artifact_obj, build_dir)
|
||||||
except DeployException as e:
|
except DeployException as e:
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
abort(400, e.message)
|
abort(400, e.message)
|
||||||
|
@ -133,7 +153,7 @@ def fetch_jenkins_build(job_name, build_num):
|
||||||
|
|
||||||
return jsonify({})
|
return jsonify({})
|
||||||
|
|
||||||
def deploy_tarball(tar_gz_url, build_dir):
|
def deploy_tarball(artifact, build_dir):
|
||||||
"""Download a tarball from jenkins and unpack it
|
"""Download a tarball from jenkins and unpack it
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
@ -145,20 +165,22 @@ def deploy_tarball(tar_gz_url, build_dir):
|
||||||
)
|
)
|
||||||
os.mkdir(build_dir)
|
os.mkdir(build_dir)
|
||||||
|
|
||||||
|
# Download the tarball here as buildkite needs auth to do this
|
||||||
|
# we don't pgp-sign buildkite artifacts, relying on HTTPS and buildkite
|
||||||
|
# not being evil. If that's not good enough for you, don't use riot.im/develop.
|
||||||
|
resp = requests.get(artifact['download_url'], stream=True, headers=req_headers())
|
||||||
|
resp.raise_for_status()
|
||||||
|
with open(artifact['filename'], 'wb') as ofp:
|
||||||
|
shutil.copyfileobj(resp.raw, ofp)
|
||||||
|
|
||||||
# we rely on the fact that flask only serves one request at a time to
|
# we rely on the fact that flask only serves one request at a time to
|
||||||
# ensure that we do not overwrite a tarball from a concurrent request.
|
# ensure that we do not overwrite a tarball from a concurrent request.
|
||||||
|
|
||||||
return deployer.deploy(tar_gz_url, build_dir)
|
return deployer.deploy(artifact['filename'], build_dir)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
|
parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
|
||||||
parser.add_argument(
|
|
||||||
"-j", "--jenkins", dest="jenkins", default="https://matrix.org/jenkins/", help=(
|
|
||||||
"The base URL of the Jenkins web server. This will be hit to get the\
|
|
||||||
built artifacts (the .gz file) for redeploying."
|
|
||||||
)
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-p", "--port", dest="port", default=4000, type=int, help=(
|
"-p", "--port", dest="port", default=4000, type=int, help=(
|
||||||
"The port to listen on for requests from Jenkins."
|
"The port to listen on for requests from Jenkins."
|
||||||
|
@ -204,13 +226,33 @@ if __name__ == "__main__":
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--webhook-token", dest="webhook_token", help=(
|
||||||
|
"Only accept pokes with this buildkite token."
|
||||||
|
), required=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--api-token", dest="api_token", help=(
|
||||||
|
"API access token for buildkite. Require read_artifacts scope."
|
||||||
|
), required=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# We require a matching webhook token, but because we take everything else
|
||||||
|
# about what to deploy from the poke body, we can be a little more paranoid
|
||||||
|
# and only accept builds / artifacts from a specific buildkite org
|
||||||
|
parser.add_argument(
|
||||||
|
"--org", dest="buildkit_org", help=(
|
||||||
|
"Lock down to this buildkite org"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.jenkins.endswith("/"): # important for urljoin
|
|
||||||
arg_jenkins_url = args.jenkins
|
|
||||||
else:
|
|
||||||
arg_jenkins_url = args.jenkins + "/"
|
|
||||||
arg_extract_path = args.extract
|
arg_extract_path = args.extract
|
||||||
arg_symlink = args.symlink
|
arg_symlink = args.symlink
|
||||||
|
arg_webbook_token = args.webhook_token
|
||||||
|
arg_api_token = args.api_token
|
||||||
|
arg_buildkit_org = args.buildkit_org
|
||||||
|
|
||||||
if not os.path.isdir(arg_extract_path):
|
if not os.path.isdir(arg_extract_path):
|
||||||
os.mkdir(arg_extract_path)
|
os.mkdir(arg_extract_path)
|
||||||
|
@ -222,25 +264,17 @@ if __name__ == "__main__":
|
||||||
for include in args.include:
|
for include in args.include:
|
||||||
deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })
|
deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })
|
||||||
|
|
||||||
|
|
||||||
# we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to
|
|
||||||
# the jenkins server (and the jenkins server not being compromised and/or
|
|
||||||
# github not serving it compromised source). If that's not good enough for
|
|
||||||
# you, don't use riot.im/develop.
|
|
||||||
deployer.verify_signature = False
|
|
||||||
|
|
||||||
if args.tarball_uri is not None:
|
if args.tarball_uri is not None:
|
||||||
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
|
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
|
||||||
deploy_tarball(args.tarball_uri, build_dir)
|
deploy_tarball(args.tarball_uri, build_dir)
|
||||||
else:
|
else:
|
||||||
print(
|
print(
|
||||||
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Include files: %s" %
|
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Include files: %s" %
|
||||||
(args.port,
|
(args.port,
|
||||||
arg_extract_path,
|
arg_extract_path,
|
||||||
" (clean after)" if deployer.should_clean else "",
|
" (clean after)" if deployer.should_clean else "",
|
||||||
arg_symlink,
|
arg_symlink,
|
||||||
arg_jenkins_url,
|
|
||||||
deployer.symlink_paths,
|
deployer.symlink_paths,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
app.run(host="0.0.0.0", port=args.port, debug=True)
|
app.run(port=args.port, debug=False)
|
||||||
|
|
Loading…
Reference in New Issue