initial commit

.gitignore

@@ -0,0 +1,2 @@
+env
+context_dir

.mandrakeignore

@@ -0,0 +1,2 @@
+env
+context_dir

Mandrake.toml

@@ -0,0 +1,31 @@
+[defaults]
+username = "alvie"
+password = "passoword"
+try-multiple = false
+run-local = true
+output = "out"
+
+[[remotes]]
+name = "a2"
+host = "alvps2"
+default = false
+
+[remotes.ssh]
+host = "alvps2"
+
+[[remotes]]
+name = "gareth"
+host = "http://gareth:5000"
+default = false
+
+[remotes.ssh]
+host = "gareth"
+
+[[remotes]]
+name = "local"
+host = "http://localhost:5000"
+default = true
+job-id = "dc1f1627-7fb9-4bca-989c-535eb79bc899"
+
+[remotes.ssh]
+host = "alvie@localhost"

config-server.toml

@@ -0,0 +1,2 @@
+docker-daemon-url = "unix:///var/run/docker.sock"
+context-dir = "/home/alvie/Documents/projects/mandrake/context_dir"

requirements.txt

@@ -0,0 +1,8 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
+docker==7.0.0
+idna==3.6
+packaging==23.2
+requests==2.31.0
+tomli==2.0.1
+urllib3==2.1.0

src/mandrake-server.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+
+import concurrent.futures
+import pathlib as pl
+import sys
+import json
+import uuid
+import traceback
+import time
+
+
+import docker
+import tomli
+
+
+from flask import Flask, request
+
+
+STATE_WAITING = "WAITING"
+STATE_CONTEXT_DELIVERED = "CONTEXT_DELIVERED"
+STATE_SUBMITTED_TO_POOL = "SUBMITTED_TO_POOL"
+STATE_BUILDING_CONTAINER = "BUILDING_CONTAINER"
+STATE_BUILD_FAILED = "BUILD_FAILED"
+STATE_RUNNING = "RUNNING"
+STATE_RUNNING_FAILED = "RUNNING_FAILED"
+STATE_FINISED = "FINISHED"
+
+
+
+def get_args():
+    """ Get command line arguments """
+
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('config', type=pl.Path)
+    return parser.parse_args()
+
+
+args = get_args()
+config = tomli.loads(args.config.read_text())
+
+def get_docker_daemon(config):
+    base_url = config.get('docker-daemon-url')
+    base_url = base_url or "unix:///var/run/docker.sock"
+
+    return docker.DockerClient(base_url=base_url)
+
+dock = get_docker_daemon(config)
+app = Flask(__name__)
+
+state = {}
+pool = concurrent.futures.ThreadPoolExecutor(max_workers=4)
+
+
+@app.route("/jobs", methods = [ "POST" ])
+def post_jobs():
+    job_id = str(uuid.uuid4())
+    job = {
+            "id": job_id,
+            "state": STATE_WAITING,
+            "context_dir":  f"{config['context-dir']}/{job_id}",
+            }
+    state[job['id']] = job
+
+    return json.dumps(job)
+
+def submit_job(job_id, params):
+    tag = f"mandrake-job-{job_id}"
+    path = f"{config['context-dir']}/{job_id}"
+    def func():
+        state[job_id]['state'] = STATE_BUILDING_CONTAINER
+        try:
+            (image, _) = dock.images.build(
+                    path=path,
+                    tag=tag,
+                    )
+        except Exception as e:
+            state[job_id]['state'] = STATE_BUILD_FAILED
+            state[job_id]['err'] = '\n'.join(traceback.format_exception(e))
+            return
+
+        state[job_id]['state'] = STATE_RUNNING
+
+        container = None
+        try:
+            container = dock.containers.run(
+                    image = image.id,
+                    volumes = {
+                        path: {
+                            'bind': '/context',
+                            'mode': 'rw',
+                            }
+                        },
+                    detach=True,
+                    **params,
+                    )
+        except Exception as e:
+            state[job_id]['state'] = STATE_RUNNING_FAILED
+            state[job_id]['err'] = '\n'.join(traceback.format_exception(e))
+            return
+
+        while True:
+            time.sleep(1)
+            container = dock.containers.get(container.id)
+            print(f"{container.status=}")
+            print( container.status not in [ 'created', 'running' ])
+            if  container.status not in [ 'created', 'running' ]:
+                break
+
+        state[job_id]['state'] = STATE_FINISED
+        output = container.attach(stdout=True, stderr=True, logs=True)
+        state[job_id]['output'] = output.decode("utf-8")
+
+    return func
+
+@app.route("/jobs/<job_id>", methods = [ "PATCH" ])
+def patch_job(job_id):
+    new_state = request.get_json()['state']
+    params = request.get_json()['params']
+
+    state[job_id]['state'] = new_state
+
+    if new_state == STATE_CONTEXT_DELIVERED:
+        state[job_id]['state'] = STATE_SUBMITTED_TO_POOL
+        pool.submit(submit_job(job_id, params))
+
+    return json.dumps(state[job_id])
+
+@app.route("/jobs")
+def get_jobs():
+    return json.dumps(state)
+
+
+@app.route("/jobs/<job_id>")
+def get_job(job_id):
+    return json.dumps(state[job_id])
+
+if __name__ == "__main__":
+    app.run(debug=True)

src/mandrake.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+
+import sys
+import requests
+import time
+
+import tomli
+import tomli_w
+import sysrsync as rsync
+from pathlib import Path
+
+STATE_WAITING = "WAITING"
+STATE_CONTEXT_DELIVERED = "CONTEXT_DELIVERED"
+STATE_SUBMITTED_TO_POOL = "SUBMITTED_TO_POOL"
+STATE_BUILDING_CONTAINER = "BUILDING_CONTAINER"
+STATE_BUILD_FAILED = "BUILD_FAILED"
+STATE_RUNNING = "RUNNING"
+STATE_RUNNING_FAILED = "RUNNING_FAILED"
+STATE_FINISED = "FINISHED"
+
+
+def get_args():
+    """ Get command line arguments """
+
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-m', '--mandrakefile', type=Path, default=Path('Mandrake.toml'))
+    parser.add_argument('-i', '--ignorefile', type=Path, default=Path('.mandrakeignore'))
+    parser.add_argument('-r', '--remote', type=str, default=None)
+    parser.add_argument('-f', '--force', action='store_true')
+    parser.add_argument('-c', '--create', action='store_true')
+    parser.add_argument('command', nargs="+")
+    return parser.parse_args()
+
+
+def get_remote(config, args):
+    if len(config['remotes']) < 1:
+        raise ValueError(f"No remotes defined in {args.mandrakefile}")
+
+    if args.remote is not None:
+        for remote in config['remotes']:
+            if remote['name'] == args.remote:
+                return remote
+
+        raise ValueError(f"Remote '{args.remote}' not found in config")
+
+    for remote in config['remotes']:
+        if remote['default']:
+            return remote
+
+    return config['remotes'][0]
+
+
+def create_job(remote):
+    print(f"{remote['host']}/jobs")
+    return requests.post(f"{remote['host']}/jobs").json()
+
+def send_context(remote, job, ignorefile):
+    # TODO ignore files in .dockerignore and .gitignore
+    options = ['--recursive', '--update', '-v']
+    if ignorefile.exists():
+        options.append(f"--exclude-from={ignorefile}")
+
+    rsync.run(
+            source=".",
+            destination=job['context_dir'],
+            destination_ssh=remote['ssh']['host'],
+            options=options,
+            )
+
+
+def sync_output_folder(remote, job, output_folder):
+    options = ['--recursive', '--update', '-v']
+    source_folder = f"{job['context_dir']}/{output_folder}"
+    destination_folder = output_folder
+    
+    rsync.run(
+            source = source_folder,
+            source_ssh = remote['ssh']['host'],
+            destination = destination_folder,
+            options = options,
+            )
+
+
+def get_job_status(remote, job_id):
+    return requests.get(f"{remote['host']}/jobs/{job_id}").json()
+
+
+def send_command(remote, job, command):
+    requests.patch(f"{remote['host']}/jobs/{job['id']}", json = {
+        'state': STATE_CONTEXT_DELIVERED,
+        'params': {
+            'command': command
+            }
+        })
+
+
+def main(args):
+    """ Entry point for script """
+    print("Reading config...")
+    config = tomli.loads(args.mandrakefile.read_text())
+    print(f"{config =}")
+    remote = get_remote(config, args)
+    print(f"{remote =}")
+    print("Detecting job or creating job...")
+    job_id = remote.get('job-id')
+    if job_id is None or args.create:
+        print("Job not present in config for remote. Creating...")
+        job_id = create_job(remote)['id']
+        print(f"Saving job_id to {args.mandrakefile}")
+        remote['job-id'] = job_id
+        args.mandrakefile.write_text(tomli_w.dumps(config))
+
+    print("Retreiving job status...")
+    job = get_job_status(remote, job_id)
+    oldstate = None
+    print(f"{job =}")
+    while job['state'] not in [ STATE_BUILD_FAILED, STATE_RUNNING_FAILED, STATE_FINISED, STATE_WAITING ]:
+        if args.force:
+            break
+
+        job = get_job_status(remote, job_id)
+        if job != oldstate:
+            print(f"Job already in progress (pass -f to force): {job =}")
+            oldstate = job
+
+    print("Sending context via rsync...")
+    send_context(remote, job, args.ignorefile)
+    print(f"Executing command on remote server: {args.command}")
+    send_command(remote, job, args.command)
+
+    while True:
+        job = get_job_status(remote, job_id)
+        if job != oldstate:
+            print(f"{job=}")
+            oldstate = job
+
+        if job['state'] in [ STATE_BUILD_FAILED, STATE_RUNNING_FAILED, STATE_FINISED ]:
+            if 'err' in job.keys():
+                print(job['err'])
+                return 1
+
+        if job['state'] == STATE_FINISED:
+            sync_output_folder(remote, job, config['defaults']['output'])
+            print(f"{job=}")
+            return 0
+
+        time.sleep(1)
+
+    return 0
+
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main(get_args()))
+    except KeyboardInterrupt:
+        sys.exit(0)