From 0880eee6ee841e41d5ad29083e371213286d4991 Mon Sep 17 00:00:00 2001 From: Akbar Rahman Date: Sun, 24 Dec 2023 20:25:33 +0000 Subject: [PATCH] initial commit --- .gitignore | 2 + .mandrakeignore | 2 + Mandrake.toml | 31 ++++++++ config-server.toml | 2 + requirements.txt | 8 +++ src/mandrake-server.py | 140 ++++++++++++++++++++++++++++++++++++ src/mandrake.py | 157 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 342 insertions(+) create mode 100644 .gitignore create mode 100644 .mandrakeignore create mode 100644 Mandrake.toml create mode 100644 config-server.toml create mode 100644 requirements.txt create mode 100755 src/mandrake-server.py create mode 100755 src/mandrake.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ecc5025 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +env +context_dir diff --git a/.mandrakeignore b/.mandrakeignore new file mode 100644 index 0000000..ecc5025 --- /dev/null +++ b/.mandrakeignore @@ -0,0 +1,2 @@ +env +context_dir diff --git a/Mandrake.toml b/Mandrake.toml new file mode 100644 index 0000000..9d29e68 --- /dev/null +++ b/Mandrake.toml @@ -0,0 +1,31 @@ +[defaults] +username = "alvie" +password = "passoword" +try-multiple = false +run-local = true +output = "out" + +[[remotes]] +name = "a2" +host = "alvps2" +default = false + +[remotes.ssh] +host = "alvps2" + +[[remotes]] +name = "gareth" +host = "http://gareth:5000" +default = false + +[remotes.ssh] +host = "gareth" + +[[remotes]] +name = "local" +host = "http://localhost:5000" +default = true +job-id = "dc1f1627-7fb9-4bca-989c-535eb79bc899" + +[remotes.ssh] +host = "alvie@localhost" diff --git a/config-server.toml b/config-server.toml new file mode 100644 index 0000000..ebf1a09 --- /dev/null +++ b/config-server.toml @@ -0,0 +1,2 @@ +docker-daemon-url = "unix:///var/run/docker.sock" +context-dir = "/home/alvie/Documents/projects/mandrake/context_dir" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2666b42 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +certifi==2023.11.17 +charset-normalizer==3.3.2 +docker==7.0.0 +idna==3.6 +packaging==23.2 +requests==2.31.0 +tomli==2.0.1 +urllib3==2.1.0 diff --git a/src/mandrake-server.py b/src/mandrake-server.py new file mode 100755 index 0000000..1d2fcad --- /dev/null +++ b/src/mandrake-server.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + + +import concurrent.futures +import pathlib as pl +import sys +import json +import uuid +import traceback +import time + + +import docker +import tomli + + +from flask import Flask, request + + +STATE_WAITING = "WAITING" +STATE_CONTEXT_DELIVERED = "CONTEXT_DELIVERED" +STATE_SUBMITTED_TO_POOL = "SUBMITTED_TO_POOL" +STATE_BUILDING_CONTAINER = "BUILDING_CONTAINER" +STATE_BUILD_FAILED = "BUILD_FAILED" +STATE_RUNNING = "RUNNING" +STATE_RUNNING_FAILED = "RUNNING_FAILED" +STATE_FINISED = "FINISHED" + + + +def get_args(): + """ Get command line arguments """ + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('config', type=pl.Path) + return parser.parse_args() + + +args = get_args() +config = tomli.loads(args.config.read_text()) + +def get_docker_daemon(config): + base_url = config.get('docker-daemon-url') + base_url = base_url or "unix:///var/run/docker.sock" + + return docker.DockerClient(base_url=base_url) + +dock = get_docker_daemon(config) +app = Flask(__name__) + +state = {} +pool = concurrent.futures.ThreadPoolExecutor(max_workers=4) + + +@app.route("/jobs", methods = [ "POST" ]) +def post_jobs(): + job_id = str(uuid.uuid4()) + job = { + "id": job_id, + "state": STATE_WAITING, + "context_dir": f"{config['context-dir']}/{job_id}", + } + state[job['id']] = job + + return json.dumps(job) + +def submit_job(job_id, params): + tag = f"mandrake-job-{job_id}" + path = f"{config['context-dir']}/{job_id}" + def func(): + state[job_id]['state'] = STATE_BUILDING_CONTAINER + try: + (image, _) = dock.images.build( + path=path, + tag=tag, + ) + except Exception as e: + state[job_id]['state'] = STATE_BUILD_FAILED + state[job_id]['err'] = '\n'.join(traceback.format_exception(e)) + return + + state[job_id]['state'] = STATE_RUNNING + + container = None + try: + container = dock.containers.run( + image = image.id, + volumes = { + path: { + 'bind': '/context', + 'mode': 'rw', + } + }, + detach=True, + **params, + ) + except Exception as e: + state[job_id]['state'] = STATE_RUNNING_FAILED + state[job_id]['err'] = '\n'.join(traceback.format_exception(e)) + return + + while True: + time.sleep(1) + container = dock.containers.get(container.id) + print(f"{container.status=}") + print( container.status not in [ 'created', 'running' ]) + if container.status not in [ 'created', 'running' ]: + break + + state[job_id]['state'] = STATE_FINISED + output = container.attach(stdout=True, stderr=True, logs=True) + state[job_id]['output'] = output.decode("utf-8") + + return func + +@app.route("/jobs/", methods = [ "PATCH" ]) +def patch_job(job_id): + new_state = request.get_json()['state'] + params = request.get_json()['params'] + + state[job_id]['state'] = new_state + + if new_state == STATE_CONTEXT_DELIVERED: + state[job_id]['state'] = STATE_SUBMITTED_TO_POOL + pool.submit(submit_job(job_id, params)) + + return json.dumps(state[job_id]) + +@app.route("/jobs") +def get_jobs(): + return json.dumps(state) + + +@app.route("/jobs/") +def get_job(job_id): + return json.dumps(state[job_id]) + +if __name__ == "__main__": + app.run(debug=True) diff --git a/src/mandrake.py b/src/mandrake.py new file mode 100755 index 0000000..969963f --- /dev/null +++ b/src/mandrake.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 + +import sys +import requests +import time + +import tomli +import tomli_w +import sysrsync as rsync +from pathlib import Path + +STATE_WAITING = "WAITING" +STATE_CONTEXT_DELIVERED = "CONTEXT_DELIVERED" +STATE_SUBMITTED_TO_POOL = "SUBMITTED_TO_POOL" +STATE_BUILDING_CONTAINER = "BUILDING_CONTAINER" +STATE_BUILD_FAILED = "BUILD_FAILED" +STATE_RUNNING = "RUNNING" +STATE_RUNNING_FAILED = "RUNNING_FAILED" +STATE_FINISED = "FINISHED" + + +def get_args(): + """ Get command line arguments """ + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--mandrakefile', type=Path, default=Path('Mandrake.toml')) + parser.add_argument('-i', '--ignorefile', type=Path, default=Path('.mandrakeignore')) + parser.add_argument('-r', '--remote', type=str, default=None) + parser.add_argument('-f', '--force', action='store_true') + parser.add_argument('-c', '--create', action='store_true') + parser.add_argument('command', nargs="+") + return parser.parse_args() + + +def get_remote(config, args): + if len(config['remotes']) < 1: + raise ValueError(f"No remotes defined in {args.mandrakefile}") + + if args.remote is not None: + for remote in config['remotes']: + if remote['name'] == args.remote: + return remote + + raise ValueError(f"Remote '{args.remote}' not found in config") + + for remote in config['remotes']: + if remote['default']: + return remote + + return config['remotes'][0] + + +def create_job(remote): + print(f"{remote['host']}/jobs") + return requests.post(f"{remote['host']}/jobs").json() + +def send_context(remote, job, ignorefile): + # TODO ignore files in .dockerignore and .gitignore + options = ['--recursive', '--update', '-v'] + if ignorefile.exists(): + options.append(f"--exclude-from={ignorefile}") + + rsync.run( + source=".", + destination=job['context_dir'], + destination_ssh=remote['ssh']['host'], + options=options, + ) + + +def sync_output_folder(remote, job, output_folder): + options = ['--recursive', '--update', '-v'] + source_folder = f"{job['context_dir']}/{output_folder}" + destination_folder = output_folder + + rsync.run( + source = source_folder, + source_ssh = remote['ssh']['host'], + destination = destination_folder, + options = options, + ) + + +def get_job_status(remote, job_id): + return requests.get(f"{remote['host']}/jobs/{job_id}").json() + + +def send_command(remote, job, command): + requests.patch(f"{remote['host']}/jobs/{job['id']}", json = { + 'state': STATE_CONTEXT_DELIVERED, + 'params': { + 'command': command + } + }) + + +def main(args): + """ Entry point for script """ + print("Reading config...") + config = tomli.loads(args.mandrakefile.read_text()) + print(f"{config =}") + remote = get_remote(config, args) + print(f"{remote =}") + print("Detecting job or creating job...") + job_id = remote.get('job-id') + if job_id is None or args.create: + print("Job not present in config for remote. Creating...") + job_id = create_job(remote)['id'] + print(f"Saving job_id to {args.mandrakefile}") + remote['job-id'] = job_id + args.mandrakefile.write_text(tomli_w.dumps(config)) + + print("Retreiving job status...") + job = get_job_status(remote, job_id) + oldstate = None + print(f"{job =}") + while job['state'] not in [ STATE_BUILD_FAILED, STATE_RUNNING_FAILED, STATE_FINISED, STATE_WAITING ]: + if args.force: + break + + job = get_job_status(remote, job_id) + if job != oldstate: + print(f"Job already in progress (pass -f to force): {job =}") + oldstate = job + + print("Sending context via rsync...") + send_context(remote, job, args.ignorefile) + print(f"Executing command on remote server: {args.command}") + send_command(remote, job, args.command) + + while True: + job = get_job_status(remote, job_id) + if job != oldstate: + print(f"{job=}") + oldstate = job + + if job['state'] in [ STATE_BUILD_FAILED, STATE_RUNNING_FAILED, STATE_FINISED ]: + if 'err' in job.keys(): + print(job['err']) + return 1 + + if job['state'] == STATE_FINISED: + sync_output_folder(remote, job, config['defaults']['output']) + print(f"{job=}") + return 0 + + time.sleep(1) + + return 0 + + +if __name__ == '__main__': + try: + sys.exit(main(get_args())) + except KeyboardInterrupt: + sys.exit(0)