diff --git a/stage6-core/04-docker-compose-essentials/00-run.sh b/stage6-core/04-docker-compose-essentials/00-run.sh index bba302487e0f537012d4fd81eeefbe5ae4fae860..aab992db717b633cc22c4cf81282298591885b11 100755 --- a/stage6-core/04-docker-compose-essentials/00-run.sh +++ b/stage6-core/04-docker-compose-essentials/00-run.sh @@ -31,11 +31,14 @@ sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services|g" " -# adjust services directories in start-and-disable.sh +# adjust services directories in scripts sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/start-and-disable.sh" sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/regenerate-services.sh" sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/create_backup.sh" sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/restore_from_backup.sh" +# adjust admin directories in scripts +sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|/home/${FIRST_USER_NAME}/Admin/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/start-and-disable.sh" +sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|/home/${FIRST_USER_NAME}/Admin/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/regenerate-services.sh" # enable service diff --git a/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh b/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh old mode 100644 new mode 100755 index dd543046b9ba45176c102cd01d1abeff48ccd31f..01214c41fdf3eabb1dc5e1e300d1e0dbb82a8bc7 --- a/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh +++ b/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh @@ -1,31 +1,46 @@ #!/bin/bash -e -# regenerate only given (or user if none given) stack +# WARNING: for admin stack: if needed to use a non standard image tag or gitlab repo instead of dockerhub +# set PIER_TAG or DOCKER_REGISTRY in ~/Services/.env before running this script +# regenerate only given stack (or user if none given) STACK=${1:-user} -pushd REPLACE_WITH_SERVICES_DIRECTORY +[ $STACK == "user" ] || [ $STACK == "admin" ] || [ $STACK == "critical" ] || (echo "usage: $0 [critical|admin|user] (default=user)" && exit 1) + +if [ $STACK == "admin" ]; +then +# download latest list of authorized commands from pier-admin project +# TODOS: replace develop with master for production + AUTH_CMDS_URL="https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands" + wget -O REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands $AUTH_CMDS_URL +# then restart pipe-server + systemctl restart pier-adm-pipe.service +fi + +cd REPLACE_WITH_SERVICES_DIRECTORY pushd Generated/$STACK docker-compose down popd # Update pier-services from the gitlab repo -# Warning: branch should be set +# Warning: branch should be set first pushd pier-services git pull popd -./pier-services/docker-compose-generator.py --$STACK - +./pier-services/docker-compose-generator.py --$STACK # force rescan db in case of new database creation before restarting services -[ $STACK != "critical" ] && docker exec db /usr/local/bin/check-create-db.sh +[ $STACK == "user" ] && docker exec db /usr/local/bin/check-create-db.sh pushd Generated/$STACK # need to explicitely pull for updating existing images docker-compose pull docker-compose up -d +# clean old images and containers +docker container prune -f docker image prune -f popd \ No newline at end of file diff --git a/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh b/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh index c42f767976fa1bccc5b978bbcc29e0764d600c02..d4ddf091b3258e6b605a061c01eb8f9e24ba8b47 100755 --- a/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh +++ b/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh @@ -10,6 +10,11 @@ echo "services:" > config.yml pushd pier-services git pull popd +# download latest list of authorized commands from pier-admin project +# TODOS: replace develop with master for production +AUTH_CMDS_URL="https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands" +wget -O REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands $AUTH_CMDS_URL + ./pier-services/docker-compose-generator.py # create docker networks for critical services diff --git a/stage6-core/05-pier-admin/00-packages b/stage6-core/05-pier-admin/00-packages deleted file mode 100644 index 5664e303b5dc2e9ef8e14a0845d9486ec1920afd..0000000000000000000000000000000000000000 --- a/stage6-core/05-pier-admin/00-packages +++ /dev/null @@ -1 +0,0 @@ -git diff --git a/stage6-core/05-pier-admin/00-pip3 b/stage6-core/05-pier-admin/00-pip3 new file mode 100644 index 0000000000000000000000000000000000000000..433774c847e14192426df748d68e6e59958ea8da --- /dev/null +++ b/stage6-core/05-pier-admin/00-pip3 @@ -0,0 +1 @@ +pyjwt diff --git a/stage6-core/05-pier-admin/01-run.sh b/stage6-core/05-pier-admin/01-run.sh index 01bf1fd3cabad0e7849b78fc2a5ba9599bd13502..72b7b63171284607888f6627ccb0aba359f7c71a 100755 --- a/stage6-core/05-pier-admin/01-run.sh +++ b/stage6-core/05-pier-admin/01-run.sh @@ -8,57 +8,43 @@ RADM_DIR="${ROOTFS_DIR}${ADM_DIR}" mkdir -p "${RADM_DIR}" SERVICES_DIR=${PIER_HOME}/Services +RSERVICES_DIR=${ROOTFS_DIR}${SERVICES_DIR} + GENERATED_DIR=${SERVICES_DIR}/Generated -WSD_DIR=${ADM_DIR}/websocketd -RWSD_DIR=${RADM_DIR}/websocketd -mkdir -p ${RWSD_DIR} +PIPE_DIR=${SERVICES_DIR}/pipes +RPIPE_DIR=${RSERVICES_DIR}/pipes +mkdir -p ${RPIPE_DIR} SECRETS_DIR=${PIER_HOME}/Services/Secrets RSECRETS_DIR=${ROOTFS_DIR}/${SECRETS_DIR} mkdir -p ${RSECRETS_DIR} # make sure it exists at boot time +mkdir -p ${RPIPE_DIR} -#1. Install websocketd (https://github.com/joewalnes/websocketd), a websocket daemon. - # figure out latest websocketd URL -LATEST_WSD_VER=$(curl -s https://api.github.com/repos/joewalnes/websocketd/releases/latest | grep "tag_name" | awk '{print substr($2, 3, length($2)-4)}') -LATEST_WSD_URL="https://github.com/joewalnes/websocketd/releases/download/v${LATEST_WSD_VER}/websocketd-${LATEST_WSD_VER}-linux_arm.zip" -TMPZIP=$(mktemp --suffix .zip) -TMPWSD=$(mktemp -d) -curl -o ${TMPZIP} -L --retry 5 ${LATEST_WSD_URL}; -unzip -o -d ${TMPWSD} ${TMPZIP} websocketd -rm ${TMPZIP} -install -m 755 ${TMPWSD}/websocketd "${RWSD_DIR}/websocketd" -rm -rf ${TMPZIP} ${TMPWSD} +##################################### SECURITY WARNING ################################## +# install pipe-server in admin dir which is **not** mounted in the pier admin container for better protection +# Authorized commands and config.json files must also be in admin for same reason +# Pipes themselves and log file must be in the mounted dir so that the container can interact with them +######################################################################################### # the server that will be run by the daemon -install -m 755 files/websocket-server.py "${RWSD_DIR}/websocket-server.py" -sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${ADM_DIR}/websocketd|g" "${RWSD_DIR}/websocket-server.py" -sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|${SERVICES_DIR}|g" "${RWSD_DIR}/websocket-server.py" +install -m 755 files/pipe-server.py "${RADM_DIR}/pipe-server.py" +sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/pipe-server.py" # the server config -install -m 644 files/config.json "${RWSD_DIR}/config.json" -sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${WSD_DIR}|g" "${RWSD_DIR}/config.json" -sed -i "s|REPLACE_WITH_SECRETS_DIR|${SECRETS_DIR}|g" "${RWSD_DIR}/config.json" +install -m 644 files/config.json "${RADM_DIR}/config.json" +sed -i "s|REPLACE_WITH_PIPE_DIRECTORY|${PIPE_DIR}|g" "${RADM_DIR}/config.json" +sed -i "s|REPLACE_WITH_SECRETS_DIR|${SECRETS_DIR}|g" "${RADM_DIR}/config.json" +sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/config.json" # install the service and the script that will run the daemon -install -m 644 files/websocketd.service "${ROOTFS_DIR}/etc/systemd/system/websocketd.service" -sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${ROOTFS_DIR}/etc/systemd/system/websocketd.service" - -install -m 755 files/start-websocketd.sh "${RADM_DIR}/start-websocketd.sh" -sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${WSD_DIR}|g" "${RADM_DIR}/start-websocketd.sh" - -# install the scripts which updates the admin services -install -m 755 files/update-admin.sh "${RADM_DIR}/update-admin.sh" -sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/update-admin.sh" -sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${WSD_DIR}|g" "${RADM_DIR}/update-admin.sh" -sed -i "s|REPLACE_WITH_GEN_DIRECTORY|${GENERATED_DIR}|g" "${RADM_DIR}/update-admin.sh" -sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|${SERVICES_DIR}|g" "${RADM_DIR}/update-admin.sh" +install -m 644 files/pier-adm-pipe.service "${ROOTFS_DIR}/etc/systemd/system/pier-adm-pipe.service" +sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${ROOTFS_DIR}/etc/systemd/system/pier-adm-pipe.service" install -m 755 files/update-system.sh "${RADM_DIR}/update-system.sh" -log "enabling service for websocketd" +log "enabling service for pier-adm-pipe" on_chroot << EOF - pip3 install --retries 3 --no-input pyjwt - systemctl enable websocketd + systemctl enable pier-adm-pipe EOF diff --git a/stage6-core/05-pier-admin/files/config.json b/stage6-core/05-pier-admin/files/config.json index 00ec5b977cd0cff339b3d4920c4cfd55561e33a8..e90c56181ce232a6951db299337159eeb717a4db 100644 --- a/stage6-core/05-pier-admin/files/config.json +++ b/stage6-core/05-pier-admin/files/config.json @@ -1,5 +1,8 @@ { - "log_level": "debug", - "jwt_key_file": "REPLACE_WITH_SECRETS_DIR/WEBSOCKETD_KEY", - "authorized_commands_file": "REPLACE_WITH_WSD_DIRECTORY/authorized_commands" -} \ No newline at end of file + "log_level": "info", + "jwt_key_file": "REPLACE_WITH_SECRETS_DIR/PIPE_JWT_KEY", + "authorized_commands_file": "REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands", + "cmd_pipe": "REPLACE_WITH_PIPE_DIRECTORY/command_pipe", + "out_pipe": "REPLACE_WITH_PIPE_DIRECTORY/output_pipe", + "log_file": "REPLACE_WITH_PIPE_DIRECTORY/pipe.log" +} diff --git a/stage6-core/05-pier-admin/files/websocketd.service b/stage6-core/05-pier-admin/files/pier-adm-pipe.service similarity index 58% rename from stage6-core/05-pier-admin/files/websocketd.service rename to stage6-core/05-pier-admin/files/pier-adm-pipe.service index 7437611f33945145afac3fc122b69ade1ae8a4b8..3e40dfd47d25c9470b83b182621fcbf9808e81c6 100644 --- a/stage6-core/05-pier-admin/files/websocketd.service +++ b/stage6-core/05-pier-admin/files/pier-adm-pipe.service @@ -1,9 +1,9 @@ [Unit] -Description=Websocketd service +Description=Pier Admin Pipe service After=network-online.target local-fs.target [Service] -ExecStart=REPLACE_WITH_ADMIN_DIRECTORY/start-websocketd.sh +ExecStart=REPLACE_WITH_ADMIN_DIRECTORY/pipe-server.py Restart=always RestartSec=2 diff --git a/stage6-core/05-pier-admin/files/pipe-server.py b/stage6-core/05-pier-admin/files/pipe-server.py new file mode 100755 index 0000000000000000000000000000000000000000..ae78d7106c158cdb3d9180bf77f5dbff7cb7558d --- /dev/null +++ b/stage6-core/05-pier-admin/files/pipe-server.py @@ -0,0 +1,214 @@ +#!/usr/bin/python3 +# File: pipe-server.py +# server for executing "bare metal" commands in pier admin +# Author: Olivier LEVILLAIN +# January 03, 2021 +import threading +import json +import subprocess +import logging +import traceback +from typing import Dict +import jwt +import pprint +import os +import errno +import tarfile +import re + +pipe_server_dir="REPLACE_WITH_ADMIN_DIRECTORY" +ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + +def usage(): + print ("$ cd %s && ./pipe-server.py&" % pipe_server_dir) + +class Config: + def __init__(self, cfg_file = pipe_server_dir + '/config.json'): + self.cmd_pipe = self.out_pipe = None + with open(cfg_file) as f: + self.__dict__ = json.load(f) + if self.__dict__.get('jwt_key_file') == None: raise Exception("No JWT key file in config.json") + with open(self.jwt_key_file) as f: + self.jwt_key = f.read() + if self.__dict__.get('log_level') == None: self.log_level = logging.WARNING + elif self.log_level == 'debug': self.log_level = logging.DEBUG + elif self.log_level == 'info': self.log_level = logging.INFO + elif self.log_level == 'warning': self.log_level = logging.WARNING + elif self.log_level == 'error': self.log_level = logging.ERROR + elif self.log_level == 'critical': self.log_level = logging.CRITICAL + else: assert False, "log level %s unknown" % self.log_level + logger = logging.getLogger() + logger.setLevel(self.log_level) + if self.__dict__.get('log_file') == None: self.log_file = None + # tar gzip of the log file then empty it + if self.log_file and os.path.exists(self.log_file) and os.path.getsize(self.log_file) > 0: + tgz = self.log_file + '.tgz' + for i in range(8,0,-1): + p1 = self.log_file + '.' + str(i) + '.tgz' + p2 = self.log_file + '.' + str(i+1) + '.tgz' + if os.path.exists(p1): os.rename(p1, p2) + if os.path.exists(tgz): os.rename(tgz, self.log_file + '.1.tgz') + tar = tarfile.open(self.log_file + ".tgz", "w:gz") + tar.add(self.log_file) + tar.close() + # empty log file + with open(self.log_file, 'w') as fl: fl.close() + ch = logging.FileHandler(self.log_file) if self.log_file else logging.StreamHandler() + ch.setLevel(self.log_level) + ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) + logger.addHandler(ch) + if self.__dict__.get('cmd_pipe') == None: raise Exception("No command pipe in config.json") + if self.__dict__.get('out_pipe') == None: raise Exception("No output pipe in config.json") + self.last_auth_load = 0 + self.authorized_commands = [] + self.loadAuthorizedCmds() + + def loadAuthorizedCmds(self): + # reload auth cmds only if modified + if self.__dict__.get('authorized_commands_file'): + ct = os.stat(self.authorized_commands_file).st_ctime + if ct > self.last_auth_load: + with open(self.authorized_commands_file) as f: + self.authorized_commands = json.load(f) + self.last_auth_load = ct + logging.info('authorized commands:'+pprint.pformat(self.authorized_commands)) + +class Pipes: + def __init__(self, config: Config): + self.cmd_pipe = config.cmd_pipe + self.out_pipe = config.out_pipe + # remove then recreate fifos + logging.info("recreating pipes...") + self.__unlink_pipe(self.cmd_pipe) + self.__create_pipe(self.cmd_pipe) + self.__unlink_pipe(self.out_pipe) + self.__create_pipe(self.out_pipe) + logging.info("recreating pipes...done") + def __unlink_pipe(self, path): + try: os.unlink(path) + except OSError as oe: + if oe.errno != errno.ENOENT: + logging.error(oe) + raise + def __create_pipe(self, path): + try: os.mkfifo(path) + except OSError as oe: + if oe.errno != errno.EEXIST: + logging.error(oe) + raise + def send_pipe(self, dict: Dict): + if dict.get('line'): + dict['line'] = ansi_escape.sub('', dict['line']) + elif dict.get('out'): + for i in range(len(dict['out'])): + dict['out'][i] = ansi_escape.sub('', dict['out'][i]) + res = json.dumps(dict, separators=(',', ':')) + # open output pipe for writing + with open(self.out_pipe, 'w') as out_pipe: + logging.info("return >{0}<".format(res)) + out_pipe.write(res + "\n") + out_pipe.flush() + out_pipe.close() + logging.debug("writing to pipe...done") + + def send_error(self, id = -1, err = None, status = -1, message = None): + if err: + logging.error(err) + logging.error(traceback.format_exc()) + if not message: + if err: + message = pprint.pformat(err) + else: + message = 'Unknown Error' + self.send_pipe({ "id": id, "status": status, "out": [message]}) + + def loop(self): + while True: + logging.debug("Waiting Cmd Pipe...") + with open(self.cmd_pipe) as fifo: + logging.debug("Reading Cmd Pipe...") + for line in fifo: + logging.debug('Read: <{0}>'.format(line)) + try: + l = line.strip("\n"); + cmd = Command(l) + proc = Process(cmd) + except Exception as err: + pipes.send_error(-1, err) + logging.debug("Cmd Pipe closed by writer") + +class Process: + def __init__(self, cmd): + try: + if (cmd.sync): self.sync_exec(cmd) + else: self.async_exec(cmd) + except Exception as err: + pipes.send_error(cmd.id, err) + + def sync_exec(self, cmd): + try: + proc = subprocess.run(cmd.command(), input=cmd.input, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=cmd.timeout, cwd=cmd.cwd, encoding='utf-8') + pipes.send_pipe({ "id": cmd.id, "status":proc.returncode, "out": proc.stdout.splitlines()}) + except subprocess.TimeoutExpired as err: + pipes.send_error(cmd.id, err, -100, 'Time Out Expired') + + def async_exec(self, cmd): + proc = subprocess.Popen(cmd.command(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, cwd=cmd.cwd, encoding='utf-8') + # TODO: manage timeout in command + if (cmd.input): proc.stdin.write(cmd.input.strip() + '\n') + for line in proc.stdout: + pipes.send_pipe({ "id": cmd.id, "line": line}) + proc.wait() + pipes.send_pipe({}) # kind of flush + pipes.send_pipe({ "id": cmd.id, "status": proc.returncode, "out": []}) +class Command: + """Class for defining a command to be run.""" + + def __init__(self, line): + self.cmd = None + self.token = None + self.args = [] + self.sync = True + self.input = None + self.timeout = 120 # default timeout = 2m + self.cwd = None + self.id = None + cmd = json.loads(line) + for key, val in cmd.items(): + if key == 'options': + for okey, oval in cmd['options'].items(): + self.__dict__[okey] = oval + else: self.__dict__[key] = val + logging.info("received", str(json.dumps(self.__dict__, separators=(',', ':')))) + self.check_cmd() + + def check_cmd(self): + # TODO: check also CGI variables like HTTP_ORIGIN (equiv CORS) + assert self.cmd != None, "Command not defined!" + assert self.token != None, "No token found in command" + assert self.id != None, "Command id not defined" + try: + payload = jwt.decode(self.token, config.jwt_key, algorithms=["HS256"]) + #logging.debug("jwt payload:" + pprint.pformat(payload)) + except jwt.exceptions.InvalidTokenError as err: + pipes.send_error(self.id, err, -150, 'Invalid Token') + assert False, "Invalid token" + assert payload.get('payload') and payload.get('payload') == config.jwt_key, "Wrong payload in token" + config.loadAuthorizedCmds() # be sure we reload last auth commands w/o restarting + assert self.cmd in config.authorized_commands, 'trying to execute cmd ' + self.cmd + ' in pipe server which is not allowed' + def command(self): + return [self.cmd] + self.args + +# load config +config = Config() +# create pipes +pipes = Pipes(config) + +# starts N threads to receive and execute the cmds +N=1 +logging.info ("starting pipe server...") +for i in range(N): + t = threading.Thread(target=pipes.loop) + t.start() + # suspend main thread until worker threads are done + t.join() diff --git a/stage6-core/05-pier-admin/files/start-websocketd.sh b/stage6-core/05-pier-admin/files/start-websocketd.sh deleted file mode 100755 index 1e15604b4deb61b273507a72f1e7cbeff341ef63..0000000000000000000000000000000000000000 --- a/stage6-core/05-pier-admin/files/start-websocketd.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -e - -# start the websocketd server -TRC_FILE=/var/log/websocketd.log -[ -f $TRC_FILE ] && tar czf $TRC_FILE.tgz $TRC_FILE && rm -f $TRC_FILE -# download latest list of authorized commands from pier-admin project -# TODOS: replace develop with master for production -AUTH_CMDS_URL="https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands" -wget -O REPLACE_WITH_WSD_DIRECTORY/authorized_commands $AUTH_CMDS_URL -pushd REPLACE_WITH_WSD_DIRECTORY -./websocketd --port 8088 ./websocket-server.py > $TRC_FILE 2>&1 -popd diff --git a/stage6-core/05-pier-admin/files/update-admin.sh b/stage6-core/05-pier-admin/files/update-admin.sh deleted file mode 100644 index c70945a5638a266b1cb04b039051693cf2674050..0000000000000000000000000000000000000000 --- a/stage6-core/05-pier-admin/files/update-admin.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# This sripts pulls and restarts the latest versions of the pier-admin (API&UI) containers. -# It also updates the list of commands authorized to the websocketd server - -GITLAB_REGISTRY=registry.gitlab.com/pierhost -DOCKERHUB_REGISTRY=mypier -DOCKER_REGISTRY_TYPE=dockerhub -export DOCKER_TAG=develop-latest - -function usage() { - echo "usage: $0 [-h] [-r (dockerhub|gitlab)] [-t tag]" - echo "-h shows this help and exits" - echo "-r docker registry where the image is pushed. default = ${DOCKER_REGISTRY_TYPE}" - echo "-t tag used to build and push the image. default = ${DOCKER_TAG}" - exit 1 -} - -while getopts ":hr:t:" flag -do - case "${flag}" in - h) usage;; - r) DOCKER_REGISTRY_TYPE="${OPTARG}";; - t) export DOCKER_TAG="${OPTARG}";; - *) usage;; - esac -done - -case "${DOCKER_REGISTRY_TYPE}" in - 'gitlab') export DOCKER_REGISTRY="${GITLAB_REGISTRY}";; - 'dockerhub') export DOCKER_REGISTRY="${DOCKERHUB_REGISTRY}";; - *) echo "Unknown docker registry:" ${DOCKER_REGISTRY_TYPE}; - usage;; -esac - -cd REPLACE_WITH_ADMIN_DIRECTORY - -# download latest list of authorized commands from pier-admin project -systemctl restart websocketd.service - -cd REPLACE_WITH_SERVICES_DIRECTORY -./pier-services/docker-compose-generator.py --admin - -cd REPLACE_WITH_GEN_DIRECTORY/admin -#docker-compose down -docker-compose pull -docker-compose up -d diff --git a/stage6-core/05-pier-admin/files/update-system.sh b/stage6-core/05-pier-admin/files/update-system.sh index f8aa6f36e672bdb9dbc940abf29b54aa0ef6bb49..e327dd693ae93f6b09c42225ab1a4c20c480f094 100644 --- a/stage6-core/05-pier-admin/files/update-system.sh +++ b/stage6-core/05-pier-admin/files/update-system.sh @@ -1,4 +1,4 @@ -#:/bin/bash +#!/bin/bash echo "Updating $(hostname) pier host operating system..." apt-get update diff --git a/stage6-core/05-pier-admin/files/websocket-server.py b/stage6-core/05-pier-admin/files/websocket-server.py deleted file mode 100644 index b6f0e7b79300e86c5ae23e5e97ec8fbe480ae95d..0000000000000000000000000000000000000000 --- a/stage6-core/05-pier-admin/files/websocket-server.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/python3 -# File: websocketd-server.py -# websocket server for executing "bare metal" commands in pier admin -# Author: Olivier LEVILLAIN -# January 03, 2021 -from sys import stdout, stdin -import threading -import json -import subprocess -import logging -import traceback -import jwt -import pprint - -websocket_dir="REPLACE_WITH_WSD_DIRECTORY" - -def usage(): - print ("$ cd %s/websocketd && ./websocketd --port=8088 ./websocketd-server.py" % websocket_dir) - -class Config: - def __init__(self, cfg_file = websocket_dir + '/config.json'): - with open(cfg_file) as f: - self.__dict__ = json.load(f) - if self.__dict__.get('jwt_key_file') == None: raise Exception("No JWT key file in config.json") - with open(self.jwt_key_file) as f: - self.jwt_key = f.read() - if self.__dict__.get('log_level') == None: self.log_level = logging.WARNING - elif self.log_level == 'debug': self.log_level = logging.DEBUG - elif self.log_level == 'info': self.log_level = logging.INFO - elif self.log_level == 'warning': self.log_level = logging.WARNING - elif self.log_level == 'error': self.log_level = logging.ERROR - elif self.log_level == 'critical': self.log_level = logging.CRITICAL - else: assert False, "log level %s unknown" % self.log_level - logger = logging.getLogger() - logger.setLevel(self.log_level) - if self.__dict__.get('log_file') == None: self.log_file = None - ch = logging.FileHandler(self.log_file) if self.log_file else logging.StreamHandler() - ch.setLevel(self.log_level) - ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) - logger.addHandler(ch) - if self.__dict__.get('authorized_commands_file') == None: self.authorized_commands = [] - else: - with open(self.authorized_commands_file) as f: - self.authorized_commands = json.load(f) - logging.info('authorized commands: %s', ', '.join(self.authorized_commands)) - -class CmdResult: - def __init__(self, status = -1, out = ''): - self.status = status if status else 0 - self.out = str(out).splitlines() - - def toJson(self) -> str: - return json.dumps(self.__dict__, separators=(',', ':')) - - def reply(self): - res=self.toJson() - logging.debug("websocketd returning " + res) - print(res) - stdout.flush() - - -def send_error(err = None, status = -1, message = None): - if err: - logging.error(err) - logging.error(traceback.format_exc()) - if not message: - if err: - message = pprint.pformat(err) - else: - message = 'Unknown Error' - CmdResult(status, message).reply() - -class Process: - def __init__(self, cmd): - try: - if (cmd.sync): self.sync_exec(cmd) - else: self.async_exec(cmd) - except Exception as err: - send_error(err) - - def sync_exec(self, cmd) -> CmdResult: - try: - proc = subprocess.run(cmd.command(), input=cmd.input, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=cmd.timeout, cwd=cmd.cwd, encoding='utf-8') - cmd=CmdResult(status=proc.returncode, out=proc.stdout) - cmd.reply() - except subprocess.TimeoutExpired as err: - send_error(err, -100, 'Time Out Expired') - - def async_exec(self, cmd): - proc = subprocess.Popen(cmd.command(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, cwd=cmd.cwd, encoding='utf-8') - # TODO: manage timeout in command and fix input - if (cmd.input): proc.stdin.write(cmd.input.strip() + '\n') - for line in proc.stdout: - # just send one line string back to caller - logging.debug("websocketd returning one line " + line) - print('"', line.rstrip(), '"') - stdout.flush() - cmd=CmdResult(status=proc.returncode) - cmd.reply() - -class Command: - """Class for defining a command to be run.""" - - def __init__(self, line): - self.cmd = None - self.token = None - self.args = [] - self.sync = True - self.input = None - self.cwd = None - self.timeout = 120 # default timeout = 2m - cmd = json.loads(line) - for key, val in cmd.items(): - self.__dict__[key] = val - logging.debug(str(json.dumps(self.__dict__, separators=(',', ':')))) - self.check_cmd() - - def check_cmd(self): - # TODO: check also CGI variables like HTTP_ORIGIN (equiv CORS) - assert self.cmd != None, "Command not defined!" - assert self.token != None, "No token found in command" - try: - payload = jwt.decode(self.token, config.jwt_key, algorithms=["HS256"]) - #logging.debug("jwt payload:" + pprint.pformat(payload)) - except jwt.exceptions.InvalidTokenError as err: - send_error(err, -150, 'Invalid Token') - assert False, "Invalid token" - assert payload.get('payload') and payload.get('payload') == config.jwt_key, "Wrong payload in token" - assert self.cmd in config.authorized_commands, 'trying to execute cmd ' + self.cmd + ' in websocketd which is not allowed' - - def command(self): - return [self.cmd] + self.args - -def serve(): - while True: - # receive cmds from the pier API - line = stdin.readline().strip() - if not line: break - try: - cmd = Command(line) - proc = Process(cmd) - except Exception as err: - send_error(err) - -# load config -config = Config() -# starts N threads to receive and execute the cmds -N=2 -logging.info ("starting websoketd server...") -for i in range(N): - t = threading.Thread(target=serve) - t.start() - # suspend main thread until worker threads are done - t.join() -