From 2bfa66a82fbd79028f2be9494e3b7ca56fc20976 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Mon, 24 May 2021 19:25:16 +0200
Subject: [PATCH 01/18] useless, already in docker stage

---
 stage6-core/05-pier-admin/00-packages | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 stage6-core/05-pier-admin/00-packages

diff --git a/stage6-core/05-pier-admin/00-packages b/stage6-core/05-pier-admin/00-packages
deleted file mode 100644
index 5664e30..0000000
--- a/stage6-core/05-pier-admin/00-packages
+++ /dev/null
@@ -1 +0,0 @@
-git
-- 
GitLab


From fed315487862aafaffff094a019302bd231e67e3 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Mon, 24 May 2021 19:26:22 +0200
Subject: [PATCH 02/18] adapted for pipes

---
 stage6-core/05-pier-admin/01-run.sh           |  48 ++---
 stage6-core/05-pier-admin/files/config.json   |  10 +-
 ...bsocketd.service => pier-adm-pipe.service} |   2 +-
 .../05-pier-admin/files/pipe-server.py        | 199 ++++++++++++++++++
 .../05-pier-admin/files/start-websocketd.sh   |  12 --
 .../05-pier-admin/files/update-admin.sh       |   5 +-
 .../05-pier-admin/files/websocket-server.py   | 155 --------------
 7 files changed, 228 insertions(+), 203 deletions(-)
 rename stage6-core/05-pier-admin/files/{websocketd.service => pier-adm-pipe.service} (73%)
 create mode 100755 stage6-core/05-pier-admin/files/pipe-server.py
 delete mode 100755 stage6-core/05-pier-admin/files/start-websocketd.sh
 delete mode 100644 stage6-core/05-pier-admin/files/websocket-server.py

diff --git a/stage6-core/05-pier-admin/01-run.sh b/stage6-core/05-pier-admin/01-run.sh
index 01bf1fd..0835b4b 100755
--- a/stage6-core/05-pier-admin/01-run.sh
+++ b/stage6-core/05-pier-admin/01-run.sh
@@ -8,50 +8,42 @@ RADM_DIR="${ROOTFS_DIR}${ADM_DIR}"
 mkdir -p "${RADM_DIR}"
 
 SERVICES_DIR=${PIER_HOME}/Services
+RSERVICES_DIR=${ROOTFS_DIR}${SERVICES_DIR}
+
 GENERATED_DIR=${SERVICES_DIR}/Generated
 
-WSD_DIR=${ADM_DIR}/websocketd
-RWSD_DIR=${RADM_DIR}/websocketd
-mkdir -p ${RWSD_DIR}
+PIPE_DIR=${SERVICES_DIR}/websocketd
+RPIPE_DIR=${RSERVICES_DIR}/websocketd
+mkdir -p ${RPIPE_DIR}
 
 SECRETS_DIR=${PIER_HOME}/Services/Secrets
 RSECRETS_DIR=${ROOTFS_DIR}/${SECRETS_DIR}
 
 mkdir -p ${RSECRETS_DIR}  # make sure it exists at boot time
+mkdir -p ${RPIPE_DIR}
 
-#1. Install websocketd (https://github.com/joewalnes/websocketd), a websocket daemon.
-  # figure out latest websocketd URL 
-LATEST_WSD_VER=$(curl -s https://api.github.com/repos/joewalnes/websocketd/releases/latest | grep "tag_name" | awk '{print substr($2, 3, length($2)-4)}') 
-LATEST_WSD_URL="https://github.com/joewalnes/websocketd/releases/download/v${LATEST_WSD_VER}/websocketd-${LATEST_WSD_VER}-linux_arm.zip"
-TMPZIP=$(mktemp --suffix .zip)
-TMPWSD=$(mktemp -d)
-curl -o ${TMPZIP} -L --retry 5 ${LATEST_WSD_URL};
-unzip -o -d ${TMPWSD} ${TMPZIP} websocketd
-rm ${TMPZIP}
-install -m 755 ${TMPWSD}/websocketd "${RWSD_DIR}/websocketd"
-rm -rf ${TMPZIP} ${TMPWSD}
+##################################### SECURITY WARNING ##################################
+# install pipe-server in admin dir which is **not** mounted in the pier admin container for better protection
+# Authorized commands and config.json files must also be in admin for same reason
+# Pipes themselves and log file must be in the mounted dir so that the container can interact with them
+#########################################################################################
 
 # the server that will be run by the daemon
-install -m 755 files/websocket-server.py "${RWSD_DIR}/websocket-server.py"
-sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${ADM_DIR}/websocketd|g" "${RWSD_DIR}/websocket-server.py"
-sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|${SERVICES_DIR}|g" "${RWSD_DIR}/websocket-server.py"
+install -m 755 files/pipe-server.py "${RADM_DIR}/pipe-server.py"
+sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/pipe-server.py"
 
 # the server config
-install -m 644 files/config.json "${RWSD_DIR}/config.json"
-sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${WSD_DIR}|g" "${RWSD_DIR}/config.json"
-sed -i "s|REPLACE_WITH_SECRETS_DIR|${SECRETS_DIR}|g" "${RWSD_DIR}/config.json"
+install -m 644 files/config.json "${RPIPE_DIR}/config.json"
+sed -i "s|REPLACE_WITH_PIPE_DIRECTORY|${PIPE_DIR}|g" "${RADM_DIR}/config.json"
+sed -i "s|REPLACE_WITH_SECRETS_DIR|${SECRETS_DIR}|g" "${RADM_DIR}/config.json"
+sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/config.json"
 
 # install the service and the script that will run the daemon
-install -m 644 files/websocketd.service "${ROOTFS_DIR}/etc/systemd/system/websocketd.service"
-sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${ROOTFS_DIR}/etc/systemd/system/websocketd.service"
-
-install -m 755 files/start-websocketd.sh "${RADM_DIR}/start-websocketd.sh"
-sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${WSD_DIR}|g" "${RADM_DIR}/start-websocketd.sh"
+install -m 644 files/pier-adm-pipe.service "${ROOTFS_DIR}/etc/systemd/system/pier-adm-pipe.service"
+sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${ROOTFS_DIR}/etc/systemd/system/pier-adm-pipe.service"
 
 # install the scripts which updates the admin services
 install -m 755 files/update-admin.sh "${RADM_DIR}/update-admin.sh"
-sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/update-admin.sh"
-sed -i "s|REPLACE_WITH_WSD_DIRECTORY|${WSD_DIR}|g" "${RADM_DIR}/update-admin.sh"
 sed -i "s|REPLACE_WITH_GEN_DIRECTORY|${GENERATED_DIR}|g" "${RADM_DIR}/update-admin.sh"
 sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|${SERVICES_DIR}|g" "${RADM_DIR}/update-admin.sh"
 
@@ -60,5 +52,5 @@ install -m 755 files/update-system.sh "${RADM_DIR}/update-system.sh"
 log "enabling service for websocketd"
 on_chroot << EOF
   pip3 install --retries 3 --no-input pyjwt
-  systemctl enable websocketd
+  systemctl enable pier-adm-pipe
 EOF
diff --git a/stage6-core/05-pier-admin/files/config.json b/stage6-core/05-pier-admin/files/config.json
index 00ec5b9..3707356 100644
--- a/stage6-core/05-pier-admin/files/config.json
+++ b/stage6-core/05-pier-admin/files/config.json
@@ -1,5 +1,9 @@
 {
-    "log_level": "debug",
+    "log_level": "info",
     "jwt_key_file": "REPLACE_WITH_SECRETS_DIR/WEBSOCKETD_KEY",
-    "authorized_commands_file": "REPLACE_WITH_WSD_DIRECTORY/authorized_commands"
-}
\ No newline at end of file
+    "authorized_commands_url": "https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands",
+    "authorized_commands_file": "REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands",
+    "cmd_pipe": "REPLACE_WITH_PIPE_DIRECTORY/command_pipe",
+    "out_pipe": "REPLACE_WITH_PIPE_DIRECTORY/output_pipe",
+    "log_file": "REPLACE_WITH_PIPE_DIRECTORY/pipe.log"
+}
diff --git a/stage6-core/05-pier-admin/files/websocketd.service b/stage6-core/05-pier-admin/files/pier-adm-pipe.service
similarity index 73%
rename from stage6-core/05-pier-admin/files/websocketd.service
rename to stage6-core/05-pier-admin/files/pier-adm-pipe.service
index 7437611..b581984 100644
--- a/stage6-core/05-pier-admin/files/websocketd.service
+++ b/stage6-core/05-pier-admin/files/pier-adm-pipe.service
@@ -3,7 +3,7 @@ Description=Websocketd service
 After=network-online.target local-fs.target
 
 [Service]
-ExecStart=REPLACE_WITH_ADMIN_DIRECTORY/start-websocketd.sh
+ExecStart=REPLACE_WITH_ADMIN_DIRECTORY/pipe-server.py
 Restart=always
 RestartSec=2
 
diff --git a/stage6-core/05-pier-admin/files/pipe-server.py b/stage6-core/05-pier-admin/files/pipe-server.py
new file mode 100755
index 0000000..6c86eb4
--- /dev/null
+++ b/stage6-core/05-pier-admin/files/pipe-server.py
@@ -0,0 +1,199 @@
+#!/usr/bin/python3
+# File: pipe-server.py
+# server for executing "bare metal" commands in pier admin
+# Author: Olivier LEVILLAIN
+# January 03, 2021
+from sys import stdout, stdin
+import threading
+import json
+import subprocess
+import logging
+import traceback
+from typing import Dict
+import jwt 
+import pprint
+import os
+import errno
+import tarfile
+import requests
+
+pipe_dir="REPLACE_WITH_ADMIN_DIR"
+
+def usage(): 
+    print ("$ cd %s && ./pipe-server.py&" % pipe_dir)
+
+class Config:
+    def __init__(self, cfg_file = pipe_dir + '/config.json'):
+        self.cmd_pipe = self.out_pipe = None
+        with open(cfg_file) as f:
+            self.__dict__ = json.load(f)
+        if self.__dict__.get('jwt_key_file') == None: raise Exception("No JWT key file in config.json")
+        with open(self.jwt_key_file) as f:
+            self.jwt_key = f.read()
+        if self.__dict__.get('log_level') == None: self.log_level = logging.WARNING
+        elif self.log_level == 'debug': self.log_level = logging.DEBUG
+        elif self.log_level == 'info': self.log_level = logging.INFO
+        elif self.log_level == 'warning': self.log_level = logging.WARNING
+        elif self.log_level == 'error': self.log_level = logging.ERROR
+        elif self.log_level == 'critical': self.log_level = logging.CRITICAL
+        else: assert False, "log level %s unknown" % self.log_level
+        logger = logging.getLogger()
+        logger.setLevel(self.log_level)
+        if self.__dict__.get('log_file') == None: self.log_file = None
+        # tar gzip of the log file then empty it
+        if self.log_file and os.path.exists(self.log_file) and os.path.getsize(self.log_file) > 0:
+            tgz = self.log_file + '.tgz'
+            for i in range(8,0,-1):
+                p1 = self.log_file + '.' + str(i) + '.tgz'
+                p2 = self.log_file + '.' + str(i+1) + '.tgz'
+                if os.path.exists(p1): os.rename(p1, p2)
+            if os.path.exists(tgz): os.rename(tgz, self.log_file + '.1.tgz')
+            tar = tarfile.open(self.log_file + "tgz", "w:gz")
+            tar.add(self.log_file)
+            tar.close()
+            # empty log file
+            with open(self.log_file, 'w') as fl: fl.close()
+        ch = logging.FileHandler(self.log_file) if self.log_file else logging.StreamHandler()
+        ch.setLevel(self.log_level)
+        ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+        logger.addHandler(ch)
+        if self.__dict__.get('authorized_commands_file') == None: self.authorized_commands = []
+        else:
+            # download authorized commands
+            if self.__dict__.get("authorized_commands_url"):
+                r = requests.get(self.authorized_commands_url, allow_redirects=True)
+                open(self.authorized_commands_file, 'wb').write(r.content)
+            # else we use the local file
+            with open(self.authorized_commands_file) as f:
+                self.authorized_commands = json.load(f)
+        logging.info('authorized commands:'+pprint.pformat(self.authorized_commands))
+        if self.__dict__.get('cmd_pipe') == None: raise Exception("No command pipe in config.json")
+        if self.__dict__.get('out_pipe') == None: raise Exception("No output pipe in config.json")
+
+class Pipes:
+    def __init__(self, config: Config):
+        self.cmd_pipe = config.cmd_pipe
+        self.out_pipe = config.out_pipe
+        # remove then recreate fifos
+        logging.info("recreating pipes...")
+        try:
+            os.unlink(self.cmd_pipe)
+            os.mkfifo(self.cmd_pipe)
+            os.unlink(self.out_pipe)
+            os.mkfifo(self.out_pipe)
+        except OSError as oe: 
+            if oe.errno != errno.EEXIST:
+                logging.error(oe)
+                raise
+        logging.info("recreating pipes...done")
+
+    def send_pipe(self, dict: Dict):
+        res = json.dumps(dict, separators=(',', ':'))
+        # open output pipe for writing
+        with open(self.out_pipe, 'w') as out_pipe:
+            logging.info("return >{0}<".format(res))
+            out_pipe.write(res + "\n")
+            out_pipe.flush()
+            out_pipe.close()
+            logging.debug("writing to pipe...done")
+
+    def send_error(self, id = -1, err = None, status = -1, message = None):
+        if err:
+            logging.error(err)
+            logging.error(traceback.format_exc())
+        if not message:
+            if err:
+                message = pprint.pformat(err)
+            else:
+                message = 'Unknown Error'
+        self.send_pipe({ "id": id, "status": status, "out": [message]})
+
+    def loop(self):
+        while True:
+            logging.debug("Waiting Cmd Pipe...")
+            with open(self.cmd_pipe) as fifo:
+                logging.debug("Reading Cmd Pipe...")
+                for line in fifo:
+                    logging.debug('Read: <{0}>'.format(line))
+                    try:
+                        l = line.strip("\n");
+                        cmd = Command(l)
+                        proc = Process(cmd)
+                    except Exception as err:
+                        pipes.send_error(-1, err)
+                logging.debug("Cmd Pipe closed by writer")
+
+class Process:
+    def __init__(self, cmd):
+        try:
+            if (cmd.sync): self.sync_exec(cmd)
+            else: self.async_exec(cmd)
+        except Exception as err:
+            pipes.send_error(cmd.id, err)
+
+    def sync_exec(self, cmd):
+        try:
+            proc = subprocess.run(cmd.command(), input=cmd.input, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=cmd.timeout, cwd=cmd.cwd, encoding='utf-8')
+            pipes.send_pipe({ "id": cmd.id, "status":proc.returncode, "out": proc.stdout.splitlines()})
+        except subprocess.TimeoutExpired as err:
+            pipes.send_error(cmd.id, err, -100, 'Time Out Expired')
+
+    def async_exec(self, cmd):
+        proc = subprocess.Popen(cmd.command(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, cwd=cmd.cwd, encoding='utf-8')
+        # TODO: manage timeout in command
+        if (cmd.input): proc.stdin.write(cmd.input.strip() + '\n')
+        for line in proc.stdout:
+            pipes.send_pipe({ "id": cmd.id, "line": line})
+        proc.wait()
+        pipes.send_pipe({})  # kind of flush
+        pipes.send_pipe({ "id": cmd.id, "status": proc.returncode, "out": []})
+class Command:
+    """Class for defining a command to be run."""
+
+    def __init__(self, line):
+        self.cmd = None
+        self.token = None
+        self.args = []
+        self.sync = True
+        self.input = None
+        self.timeout = 120  # default timeout = 2m
+        self.cwd = None
+        self.id = None
+        cmd = json.loads(line)
+        for key, val in cmd.items():
+            if key == 'options':
+                for okey, oval in cmd['options'].items():
+                    self.__dict__[okey] = oval
+            else: self.__dict__[key] = val
+        logging.info("received", str(json.dumps(self.__dict__, separators=(',', ':'))))
+        self.check_cmd()
+
+    def check_cmd(self):
+        # TODO: check also CGI variables like HTTP_ORIGIN (equiv CORS)
+        assert self.cmd != None, "Command not defined!"
+        assert self.token != None, "No token found in command"
+        assert self.id != None, "Command id not defined"
+        try:
+            payload = jwt.decode(self.token, config.jwt_key, algorithms=["HS256"])
+            #logging.debug("jwt payload:" + pprint.pformat(payload))
+        except jwt.exceptions.InvalidTokenError as err:
+            pipes.send_error(self.id, err, -150, 'Invalid Token')
+            assert False, "Invalid token"
+        assert payload.get('payload') and payload.get('payload') == config.jwt_key, "Wrong payload in token"
+        assert self.cmd in config.authorized_commands, 'trying to execute cmd ' + self.cmd + ' in websocketd which is not allowed'
+    def command(self):
+        return [self.cmd] + self.args
+
+# load config
+config = Config()
+# create pipes
+pipes = Pipes(config)
+
+# starts N threads to receive and execute the cmds
+N=1
+logging.info ("starting pipe server...")
+for i in range(N):
+    t = threading.Thread(target=pipes.loop)
+    t.start()
+    # suspend main thread until worker threads are done
+    t.join()
diff --git a/stage6-core/05-pier-admin/files/start-websocketd.sh b/stage6-core/05-pier-admin/files/start-websocketd.sh
deleted file mode 100755
index 1e15604..0000000
--- a/stage6-core/05-pier-admin/files/start-websocketd.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash -e
-
-# start the websocketd server
-TRC_FILE=/var/log/websocketd.log
-[ -f $TRC_FILE ] && tar czf $TRC_FILE.tgz $TRC_FILE && rm -f $TRC_FILE
-# download latest list of authorized commands from pier-admin project
-# TODOS: replace develop with master for production
-AUTH_CMDS_URL="https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands"
-wget -O REPLACE_WITH_WSD_DIRECTORY/authorized_commands $AUTH_CMDS_URL
-pushd REPLACE_WITH_WSD_DIRECTORY
-./websocketd --port 8088 ./websocket-server.py > $TRC_FILE 2>&1
-popd
diff --git a/stage6-core/05-pier-admin/files/update-admin.sh b/stage6-core/05-pier-admin/files/update-admin.sh
index c70945a..477fc42 100644
--- a/stage6-core/05-pier-admin/files/update-admin.sh
+++ b/stage6-core/05-pier-admin/files/update-admin.sh
@@ -33,10 +33,7 @@ case "${DOCKER_REGISTRY_TYPE}" in
      usage;;
 esac
 
-cd REPLACE_WITH_ADMIN_DIRECTORY
-
-# download latest list of authorized commands from pier-admin project
-systemctl restart websocketd.service
+systemctl restart pier-adm-pipe.service
 
 cd REPLACE_WITH_SERVICES_DIRECTORY
 ./pier-services/docker-compose-generator.py --admin
diff --git a/stage6-core/05-pier-admin/files/websocket-server.py b/stage6-core/05-pier-admin/files/websocket-server.py
deleted file mode 100644
index b6f0e7b..0000000
--- a/stage6-core/05-pier-admin/files/websocket-server.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/python3
-# File: websocketd-server.py
-# websocket server for executing "bare metal" commands in pier admin
-# Author: Olivier LEVILLAIN
-# January 03, 2021
-from sys import stdout, stdin
-import threading
-import json
-import subprocess
-import logging
-import traceback
-import jwt 
-import pprint
-
-websocket_dir="REPLACE_WITH_WSD_DIRECTORY"
-
-def usage(): 
-    print ("$ cd %s/websocketd && ./websocketd --port=8088  ./websocketd-server.py" % websocket_dir)
-
-class Config:
-    def __init__(self, cfg_file = websocket_dir + '/config.json'):
-        with open(cfg_file) as f:
-            self.__dict__ = json.load(f)
-        if self.__dict__.get('jwt_key_file') == None: raise Exception("No JWT key file in config.json")
-        with open(self.jwt_key_file) as f:
-            self.jwt_key = f.read()
-        if self.__dict__.get('log_level') == None: self.log_level = logging.WARNING
-        elif self.log_level == 'debug': self.log_level = logging.DEBUG
-        elif self.log_level == 'info': self.log_level = logging.INFO
-        elif self.log_level == 'warning': self.log_level = logging.WARNING
-        elif self.log_level == 'error': self.log_level = logging.ERROR
-        elif self.log_level == 'critical': self.log_level = logging.CRITICAL
-        else: assert False, "log level %s unknown" % self.log_level
-        logger = logging.getLogger()
-        logger.setLevel(self.log_level)
-        if self.__dict__.get('log_file') == None: self.log_file = None
-        ch = logging.FileHandler(self.log_file) if self.log_file else logging.StreamHandler()
-        ch.setLevel(self.log_level)
-        ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
-        logger.addHandler(ch)
-        if self.__dict__.get('authorized_commands_file') == None: self.authorized_commands = []
-        else: 
-            with open(self.authorized_commands_file) as f:
-                self.authorized_commands = json.load(f)
-        logging.info('authorized commands: %s', ', '.join(self.authorized_commands))
-
-class CmdResult:
-    def __init__(self, status = -1, out = ''):
-        self.status = status if status else 0
-        self.out = str(out).splitlines()
-
-    def toJson(self) -> str:
-        return json.dumps(self.__dict__, separators=(',', ':'))
-
-    def reply(self):
-        res=self.toJson()
-        logging.debug("websocketd returning " + res)
-        print(res)
-        stdout.flush()
-
-
-def send_error(err = None, status = -1, message = None):
-    if err:
-        logging.error(err)
-        logging.error(traceback.format_exc())
-    if not message:
-        if err:
-           message = pprint.pformat(err)
-        else:
-           message = 'Unknown Error' 
-    CmdResult(status, message).reply()
-
-class Process:
-    def __init__(self, cmd):
-        try:
-            if (cmd.sync): self.sync_exec(cmd)
-            else: self.async_exec(cmd)
-        except Exception as err:
-            send_error(err)
-
-    def sync_exec(self, cmd) -> CmdResult:
-        try:
-            proc = subprocess.run(cmd.command(), input=cmd.input, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=cmd.timeout, cwd=cmd.cwd, encoding='utf-8')
-            cmd=CmdResult(status=proc.returncode, out=proc.stdout)
-            cmd.reply()
-        except subprocess.TimeoutExpired as err:
-            send_error(err, -100, 'Time Out Expired')
-
-    def async_exec(self, cmd):
-        proc = subprocess.Popen(cmd.command(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, cwd=cmd.cwd, encoding='utf-8')
-        # TODO: manage timeout in command and fix input
-        if (cmd.input): proc.stdin.write(cmd.input.strip() + '\n')
-        for line in proc.stdout:
-            # just send one line string back to caller
-            logging.debug("websocketd returning one line " + line)
-            print('"', line.rstrip(), '"')
-            stdout.flush()
-        cmd=CmdResult(status=proc.returncode)
-        cmd.reply()
-
-class Command:
-    """Class for defining a command to be run."""
-
-    def __init__(self, line):
-        self.cmd = None
-        self.token = None
-        self.args = []
-        self.sync = True
-        self.input = None
-        self.cwd = None
-        self.timeout = 120  # default timeout = 2m
-        cmd = json.loads(line)
-        for key, val in cmd.items():
-            self.__dict__[key] = val
-        logging.debug(str(json.dumps(self.__dict__, separators=(',', ':'))))
-        self.check_cmd()
-
-    def check_cmd(self):
-        # TODO: check also CGI variables like HTTP_ORIGIN (equiv CORS)
-        assert self.cmd != None, "Command not defined!"
-        assert self.token != None, "No token found in command"
-        try:
-            payload = jwt.decode(self.token, config.jwt_key, algorithms=["HS256"])
-            #logging.debug("jwt payload:" + pprint.pformat(payload))
-        except jwt.exceptions.InvalidTokenError as err:
-            send_error(err, -150, 'Invalid Token')
-            assert False, "Invalid token"
-        assert payload.get('payload') and payload.get('payload') == config.jwt_key, "Wrong payload in token"
-        assert self.cmd in config.authorized_commands, 'trying to execute cmd ' + self.cmd + ' in websocketd which is not allowed'
-
-    def command(self):
-        return [self.cmd] + self.args
-
-def serve():
-    while True:
-        # receive cmds from the pier API
-        line = stdin.readline().strip()
-        if not line: break
-        try:
-            cmd = Command(line)
-            proc = Process(cmd)
-        except Exception as err:
-            send_error(err)
-
-# load config
-config = Config()
-# starts N threads to receive and execute the cmds
-N=2
-logging.info ("starting websoketd server...")
-for i in range(N):
-    t = threading.Thread(target=serve)
-    t.start()
-    # suspend main thread until worker threads are done
-    t.join()
-
-- 
GitLab


From 7aa6e40d891f17228cee0ce13fa72bec24b9f457 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Tue, 25 May 2021 16:51:51 +0200
Subject: [PATCH 03/18] fixes on dirs and doc

---
 stage6-core/05-pier-admin/01-run.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/stage6-core/05-pier-admin/01-run.sh b/stage6-core/05-pier-admin/01-run.sh
index 0835b4b..a73c6a1 100755
--- a/stage6-core/05-pier-admin/01-run.sh
+++ b/stage6-core/05-pier-admin/01-run.sh
@@ -12,8 +12,8 @@ RSERVICES_DIR=${ROOTFS_DIR}${SERVICES_DIR}
 
 GENERATED_DIR=${SERVICES_DIR}/Generated
 
-PIPE_DIR=${SERVICES_DIR}/websocketd
-RPIPE_DIR=${RSERVICES_DIR}/websocketd
+PIPE_DIR=${SERVICES_DIR}/pipes
+RPIPE_DIR=${RSERVICES_DIR}/pipes
 mkdir -p ${RPIPE_DIR}
 
 SECRETS_DIR=${PIER_HOME}/Services/Secrets
@@ -33,7 +33,7 @@ install -m 755 files/pipe-server.py "${RADM_DIR}/pipe-server.py"
 sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/pipe-server.py"
 
 # the server config
-install -m 644 files/config.json "${RPIPE_DIR}/config.json"
+install -m 644 files/config.json "${RADM_DIR}/config.json"
 sed -i "s|REPLACE_WITH_PIPE_DIRECTORY|${PIPE_DIR}|g" "${RADM_DIR}/config.json"
 sed -i "s|REPLACE_WITH_SECRETS_DIR|${SECRETS_DIR}|g" "${RADM_DIR}/config.json"
 sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/config.json"
@@ -49,7 +49,7 @@ sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|${SERVICES_DIR}|g" "${RADM_DIR}/update
 
 install -m 755 files/update-system.sh "${RADM_DIR}/update-system.sh"
 
-log "enabling service for websocketd"
+log "enabling service for pier-adm-pipe"
 on_chroot << EOF
   pip3 install --retries 3 --no-input pyjwt
   systemctl enable pier-adm-pipe
-- 
GitLab


From 50e9cdf689fc6b1b7008f6c26db76d62e8df926e Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Tue, 25 May 2021 16:52:02 +0200
Subject: [PATCH 04/18] changed desc

---
 stage6-core/05-pier-admin/files/pier-adm-pipe.service | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stage6-core/05-pier-admin/files/pier-adm-pipe.service b/stage6-core/05-pier-admin/files/pier-adm-pipe.service
index b581984..3e40dfd 100644
--- a/stage6-core/05-pier-admin/files/pier-adm-pipe.service
+++ b/stage6-core/05-pier-admin/files/pier-adm-pipe.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=Websocketd service
+Description=Pier Admin Pipe service
 After=network-online.target local-fs.target
 
 [Service]
-- 
GitLab


From 7c02d9abd262beacb5c4732ef09c83e7fe3d130f Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Tue, 25 May 2021 16:52:22 +0200
Subject: [PATCH 05/18] change name od pipe dir

---
 stage6-core/05-pier-admin/files/pipe-server.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/stage6-core/05-pier-admin/files/pipe-server.py b/stage6-core/05-pier-admin/files/pipe-server.py
index 6c86eb4..ecaa72d 100755
--- a/stage6-core/05-pier-admin/files/pipe-server.py
+++ b/stage6-core/05-pier-admin/files/pipe-server.py
@@ -17,13 +17,13 @@ import errno
 import tarfile
 import requests
 
-pipe_dir="REPLACE_WITH_ADMIN_DIR"
+pipe_server_dir="REPLACE_WITH_ADMIN_DIR"
 
 def usage(): 
-    print ("$ cd %s && ./pipe-server.py&" % pipe_dir)
+    print ("$ cd %s && ./pipe-server.py&" % pipe_server_dir)
 
 class Config:
-    def __init__(self, cfg_file = pipe_dir + '/config.json'):
+    def __init__(self, cfg_file = pipe_server_dir + '/config.json'):
         self.cmd_pipe = self.out_pipe = None
         with open(cfg_file) as f:
             self.__dict__ = json.load(f)
-- 
GitLab


From c5c8faf902485828ace9f86e5d01d3f3b54631b6 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Tue, 25 May 2021 20:01:37 +0200
Subject: [PATCH 06/18] fix create pipe when didn't exist

---
 .../05-pier-admin/files/pipe-server.py        | 24 ++++++++++++-------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/stage6-core/05-pier-admin/files/pipe-server.py b/stage6-core/05-pier-admin/files/pipe-server.py
index ecaa72d..5a252ad 100755
--- a/stage6-core/05-pier-admin/files/pipe-server.py
+++ b/stage6-core/05-pier-admin/files/pipe-server.py
@@ -48,7 +48,7 @@ class Config:
                 p2 = self.log_file + '.' + str(i+1) + '.tgz'
                 if os.path.exists(p1): os.rename(p1, p2)
             if os.path.exists(tgz): os.rename(tgz, self.log_file + '.1.tgz')
-            tar = tarfile.open(self.log_file + "tgz", "w:gz")
+            tar = tarfile.open(self.log_file + ".tgz", "w:gz")
             tar.add(self.log_file)
             tar.close()
             # empty log file
@@ -76,17 +76,23 @@ class Pipes:
         self.out_pipe = config.out_pipe
         # remove then recreate fifos
         logging.info("recreating pipes...")
-        try:
-            os.unlink(self.cmd_pipe)
-            os.mkfifo(self.cmd_pipe)
-            os.unlink(self.out_pipe)
-            os.mkfifo(self.out_pipe)
-        except OSError as oe: 
+        self.__unlink_pipe(self.cmd_pipe)
+        self.__create_pipe(self.cmd_pipe)
+        self.__unlink_pipe(self.out_pipe)
+        self.__create_pipe(self.out_pipe)
+        logging.info("recreating pipes...done")
+    def __unlink_pipe(self, path):
+        try: os.unlink(path)
+        except OSError as oe:
+            if oe.errno != errno.ENOENT:
+                logging.error(oe)
+                raise
+    def __create_pipe(self, path):
+        try: os.mkfifo(path)
+        except OSError as oe:
             if oe.errno != errno.EEXIST:
                 logging.error(oe)
                 raise
-        logging.info("recreating pipes...done")
-
     def send_pipe(self, dict: Dict):
         res = json.dumps(dict, separators=(',', ':'))
         # open output pipe for writing
-- 
GitLab


From f8d656e10cd436c2fa6ca35a84a82f6f735beced Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Tue, 25 May 2021 20:01:51 +0200
Subject: [PATCH 07/18] doc

---
 stage6-core/05-pier-admin/files/update-admin.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stage6-core/05-pier-admin/files/update-admin.sh b/stage6-core/05-pier-admin/files/update-admin.sh
index 477fc42..79d9bb3 100644
--- a/stage6-core/05-pier-admin/files/update-admin.sh
+++ b/stage6-core/05-pier-admin/files/update-admin.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # This sripts pulls and restarts the latest versions of the pier-admin (API&UI) containers.
-# It also updates the list of commands authorized to the websocketd server
+# It also restart the pier-admin-pipe service before.
 
 GITLAB_REGISTRY=registry.gitlab.com/pierhost
 DOCKERHUB_REGISTRY=mypier
-- 
GitLab


From f2e57b6a7071a5913f5a9b3d6cbec04ffb286d45 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 30 May 2021 18:53:23 +0200
Subject: [PATCH 08/18] everything in regenerate-services

---
 .../04-docker-compose-essentials/00-run.sh    |  5 ++-
 .../files/regenerate-services.sh              | 27 +++++++++---
 .../05-pier-admin/files/update-admin.sh       | 44 -------------------
 3 files changed, 25 insertions(+), 51 deletions(-)
 mode change 100644 => 100755 stage6-core/04-docker-compose-essentials/files/regenerate-services.sh
 delete mode 100644 stage6-core/05-pier-admin/files/update-admin.sh

diff --git a/stage6-core/04-docker-compose-essentials/00-run.sh b/stage6-core/04-docker-compose-essentials/00-run.sh
index bba3024..aab992d 100755
--- a/stage6-core/04-docker-compose-essentials/00-run.sh
+++ b/stage6-core/04-docker-compose-essentials/00-run.sh
@@ -31,11 +31,14 @@ sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services|g" "
 
 
 
-# adjust services directories in start-and-disable.sh
+# adjust services directories in scripts
 sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/start-and-disable.sh"
 sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/regenerate-services.sh"
 sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/create_backup.sh"
 sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|/home/${FIRST_USER_NAME}/Services/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/restore_from_backup.sh"
+# adjust admin directories in scripts
+sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|/home/${FIRST_USER_NAME}/Admin/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/start-and-disable.sh"
+sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|/home/${FIRST_USER_NAME}/Admin/|g" "${ROOTFS_DIR}/home/${FIRST_USER_NAME}/Services/regenerate-services.sh"
 
 
 # enable service
diff --git a/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh b/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh
old mode 100644
new mode 100755
index dd54304..01214c4
--- a/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh
+++ b/stage6-core/04-docker-compose-essentials/files/regenerate-services.sh
@@ -1,31 +1,46 @@
 #!/bin/bash -e
 
-# regenerate only given (or user if none given) stack
+# WARNING: for admin stack: if needed to use a non standard image tag or gitlab repo instead of dockerhub
+# set PIER_TAG or DOCKER_REGISTRY in ~/Services/.env before running this script 
 
+# regenerate only given stack (or user if none given)
 STACK=${1:-user}
 
-pushd REPLACE_WITH_SERVICES_DIRECTORY
+[ $STACK == "user" ] || [ $STACK == "admin" ] || [ $STACK == "critical" ] || (echo "usage: $0 [critical|admin|user] (default=user)" && exit 1)
+
+if [ $STACK == "admin" ];
+then
+# download latest list of authorized commands from pier-admin project
+# TODOS: replace develop with master for production
+    AUTH_CMDS_URL="https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands"
+    wget -O REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands $AUTH_CMDS_URL
+# then restart pipe-server 
+    systemctl restart pier-adm-pipe.service
+fi
+
+cd REPLACE_WITH_SERVICES_DIRECTORY
 
 pushd Generated/$STACK
 docker-compose down
 popd
 
 # Update pier-services from the gitlab repo
-# Warning: branch should be set
+# Warning: branch should be set first
 pushd pier-services
 git pull
 popd
 
-./pier-services/docker-compose-generator.py --$STACK  
-
+./pier-services/docker-compose-generator.py --$STACK
 
 # force rescan db in case of new database creation before restarting services
-[ $STACK != "critical" ] && docker exec db /usr/local/bin/check-create-db.sh
+[ $STACK == "user" ] && docker exec db /usr/local/bin/check-create-db.sh
 
 pushd Generated/$STACK
 # need to explicitely pull for updating existing images
 docker-compose pull
 docker-compose up -d
+# clean old images and containers
+docker container prune -f
 docker image prune -f
 popd
  
\ No newline at end of file
diff --git a/stage6-core/05-pier-admin/files/update-admin.sh b/stage6-core/05-pier-admin/files/update-admin.sh
deleted file mode 100644
index 79d9bb3..0000000
--- a/stage6-core/05-pier-admin/files/update-admin.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-# This sripts pulls and restarts the latest versions of the pier-admin (API&UI) containers.
-# It also restart the pier-admin-pipe service before.
-
-GITLAB_REGISTRY=registry.gitlab.com/pierhost
-DOCKERHUB_REGISTRY=mypier
-DOCKER_REGISTRY_TYPE=dockerhub
-export DOCKER_TAG=develop-latest
-
-function usage() {
-	echo "usage: $0 [-h] [-r (dockerhub|gitlab)] [-t tag]"
-	echo "-h shows this help and exits"
-	echo "-r docker registry where the image is pushed. default = ${DOCKER_REGISTRY_TYPE}"
-	echo "-t tag used to build and push the image. default = ${DOCKER_TAG}"
-	exit 1
-}
-
-while getopts ":hr:t:" flag
-do
-	case "${flag}" in
-		h) usage;;
-		r) DOCKER_REGISTRY_TYPE="${OPTARG}";;
-		t) export DOCKER_TAG="${OPTARG}";;
-		*) usage;;
-	esac
-done
-
-case "${DOCKER_REGISTRY_TYPE}" in
-  'gitlab') export DOCKER_REGISTRY="${GITLAB_REGISTRY}";;
-  'dockerhub') export DOCKER_REGISTRY="${DOCKERHUB_REGISTRY}";;
-  *) echo "Unknown docker registry:" ${DOCKER_REGISTRY_TYPE};
-     usage;;
-esac
-
-systemctl restart pier-adm-pipe.service
-
-cd REPLACE_WITH_SERVICES_DIRECTORY
-./pier-services/docker-compose-generator.py --admin
-
-cd REPLACE_WITH_GEN_DIRECTORY/admin
-#docker-compose down
-docker-compose pull 
-docker-compose up -d
-- 
GitLab


From cf056041798dfee0210b554136337514c0b086e4 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 30 May 2021 18:53:52 +0200
Subject: [PATCH 09/18] 1rst upload of authorized cmds

---
 .../04-docker-compose-essentials/files/start-and-disable.sh  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh b/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh
index c42f767..d4ddf09 100755
--- a/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh
+++ b/stage6-core/04-docker-compose-essentials/files/start-and-disable.sh
@@ -10,6 +10,11 @@ echo "services:" > config.yml
 pushd pier-services
 git pull
 popd
+# download latest list of authorized commands from pier-admin project
+# TODOS: replace develop with master for production
+AUTH_CMDS_URL="https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands"
+wget -O REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands $AUTH_CMDS_URL
+
 ./pier-services/docker-compose-generator.py
 
 # create docker networks for critical services
-- 
GitLab


From 72f9b38e0d9f1148a9c9e6ec347fb47345dd83df Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 30 May 2021 18:54:04 +0200
Subject: [PATCH 10/18] init

---
 stage6-core/05-pier-admin/00-pip3 | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 stage6-core/05-pier-admin/00-pip3

diff --git a/stage6-core/05-pier-admin/00-pip3 b/stage6-core/05-pier-admin/00-pip3
new file mode 100644
index 0000000..433774c
--- /dev/null
+++ b/stage6-core/05-pier-admin/00-pip3
@@ -0,0 +1 @@
+pyjwt
-- 
GitLab


From 73a2eb268bc23b1ce1bcf4a3b01eea68467f030f Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 30 May 2021 18:54:38 +0200
Subject: [PATCH 11/18] moved pip install to 00-pip3 file

---
 stage6-core/05-pier-admin/01-run.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stage6-core/05-pier-admin/01-run.sh b/stage6-core/05-pier-admin/01-run.sh
index a73c6a1..80bfdcf 100755
--- a/stage6-core/05-pier-admin/01-run.sh
+++ b/stage6-core/05-pier-admin/01-run.sh
@@ -51,6 +51,5 @@ install -m 755 files/update-system.sh "${RADM_DIR}/update-system.sh"
 
 log "enabling service for pier-adm-pipe"
 on_chroot << EOF
-  pip3 install --retries 3 --no-input pyjwt
   systemctl enable pier-adm-pipe
 EOF
-- 
GitLab


From 77258b988a98a5359e53992e80787ffa0f763997 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 30 May 2021 18:54:55 +0200
Subject: [PATCH 12/18] removed url for auth cmds

---
 stage6-core/05-pier-admin/files/config.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stage6-core/05-pier-admin/files/config.json b/stage6-core/05-pier-admin/files/config.json
index 3707356..1f24d08 100644
--- a/stage6-core/05-pier-admin/files/config.json
+++ b/stage6-core/05-pier-admin/files/config.json
@@ -1,7 +1,6 @@
 {
     "log_level": "info",
     "jwt_key_file": "REPLACE_WITH_SECRETS_DIR/WEBSOCKETD_KEY",
-    "authorized_commands_url": "https://gitlab.com/pierhost/pier-admin/-/raw/develop/authorized_commands",
     "authorized_commands_file": "REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands",
     "cmd_pipe": "REPLACE_WITH_PIPE_DIRECTORY/command_pipe",
     "out_pipe": "REPLACE_WITH_PIPE_DIRECTORY/output_pipe",
-- 
GitLab


From 55cb0b02d296069480bc56a9483ed3f89c927b19 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 30 May 2021 18:55:56 +0200
Subject: [PATCH 13/18] removed download of auth cmds + check new file each
 time (needed to be sure at least 1rst start)

---
 .../05-pier-admin/files/pipe-server.py        | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/stage6-core/05-pier-admin/files/pipe-server.py b/stage6-core/05-pier-admin/files/pipe-server.py
index 5a252ad..a681892 100755
--- a/stage6-core/05-pier-admin/files/pipe-server.py
+++ b/stage6-core/05-pier-admin/files/pipe-server.py
@@ -3,7 +3,7 @@
 # server for executing "bare metal" commands in pier admin
 # Author: Olivier LEVILLAIN
 # January 03, 2021
-from sys import stdout, stdin
+from sys import addaudithook, stdout, stdin
 import threading
 import json
 import subprocess
@@ -15,7 +15,6 @@ import pprint
 import os
 import errno
 import tarfile
-import requests
 
 pipe_server_dir="REPLACE_WITH_ADMIN_DIR"
 
@@ -57,18 +56,21 @@ class Config:
         ch.setLevel(self.log_level)
         ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
         logger.addHandler(ch)
-        if self.__dict__.get('authorized_commands_file') == None: self.authorized_commands = []
-        else:
-            # download authorized commands
-            if self.__dict__.get("authorized_commands_url"):
-                r = requests.get(self.authorized_commands_url, allow_redirects=True)
-                open(self.authorized_commands_file, 'wb').write(r.content)
-            # else we use the local file
-            with open(self.authorized_commands_file) as f:
-                self.authorized_commands = json.load(f)
-        logging.info('authorized commands:'+pprint.pformat(self.authorized_commands))
         if self.__dict__.get('cmd_pipe') == None: raise Exception("No command pipe in config.json")
         if self.__dict__.get('out_pipe') == None: raise Exception("No output pipe in config.json")
+        self.last_auth_load = 0
+        self.authorized_commands = []
+        self.loadAuthorizedCmds()
+        
+    def loadAuthorizedCmds(self):
+        # reload auth cmds only if modified
+        if self.__dict__.get('authorized_commands_file'):
+            ct = os.stat(self.authorized_commands_file).st_ctime
+            if ct > self.last_auth_load:
+                with open(self.authorized_commands_file) as f:
+                    self.authorized_commands = json.load(f)
+                    self.last_auth_load = ct
+                logging.info('authorized commands:'+pprint.pformat(self.authorized_commands))
 
 class Pipes:
     def __init__(self, config: Config):
@@ -186,7 +188,8 @@ class Command:
             pipes.send_error(self.id, err, -150, 'Invalid Token')
             assert False, "Invalid token"
         assert payload.get('payload') and payload.get('payload') == config.jwt_key, "Wrong payload in token"
-        assert self.cmd in config.authorized_commands, 'trying to execute cmd ' + self.cmd + ' in websocketd which is not allowed'
+        config.loadAuthorizedCmds() # be sure we reload last auth commands w/o restarting
+        assert self.cmd in config.authorized_commands, 'trying to execute cmd ' + self.cmd + ' in pipe server which is not allowed'
     def command(self):
         return [self.cmd] + self.args
 
-- 
GitLab


From f200cfd36ceeeaf1a3cc8611858976cd01625bcc Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 30 May 2021 19:01:11 +0200
Subject: [PATCH 14/18] removed install update-admin.sh

---
 stage6-core/05-pier-admin/01-run.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/stage6-core/05-pier-admin/01-run.sh b/stage6-core/05-pier-admin/01-run.sh
index 80bfdcf..72b7b63 100755
--- a/stage6-core/05-pier-admin/01-run.sh
+++ b/stage6-core/05-pier-admin/01-run.sh
@@ -42,11 +42,6 @@ sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${RADM_DIR}/config.json"
 install -m 644 files/pier-adm-pipe.service "${ROOTFS_DIR}/etc/systemd/system/pier-adm-pipe.service"
 sed -i "s|REPLACE_WITH_ADMIN_DIRECTORY|${ADM_DIR}|g" "${ROOTFS_DIR}/etc/systemd/system/pier-adm-pipe.service"
 
-# install the scripts which updates the admin services
-install -m 755 files/update-admin.sh "${RADM_DIR}/update-admin.sh"
-sed -i "s|REPLACE_WITH_GEN_DIRECTORY|${GENERATED_DIR}|g" "${RADM_DIR}/update-admin.sh"
-sed -i "s|REPLACE_WITH_SERVICES_DIRECTORY|${SERVICES_DIR}|g" "${RADM_DIR}/update-admin.sh"
-
 install -m 755 files/update-system.sh "${RADM_DIR}/update-system.sh"
 
 log "enabling service for pier-adm-pipe"
-- 
GitLab


From e5ee1d4942896ca5152ec439049620d1a021f7c2 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 6 Jun 2021 17:07:08 +0200
Subject: [PATCH 15/18] useless import + typo in admin dir

---
 stage6-core/05-pier-admin/files/pipe-server.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/stage6-core/05-pier-admin/files/pipe-server.py b/stage6-core/05-pier-admin/files/pipe-server.py
index a681892..5b78757 100755
--- a/stage6-core/05-pier-admin/files/pipe-server.py
+++ b/stage6-core/05-pier-admin/files/pipe-server.py
@@ -3,7 +3,6 @@
 # server for executing "bare metal" commands in pier admin
 # Author: Olivier LEVILLAIN
 # January 03, 2021
-from sys import addaudithook, stdout, stdin
 import threading
 import json
 import subprocess
@@ -16,7 +15,7 @@ import os
 import errno
 import tarfile
 
-pipe_server_dir="REPLACE_WITH_ADMIN_DIR"
+pipe_server_dir="REPLACE_WITH_ADMIN_DIRECTORY"
 
 def usage(): 
     print ("$ cd %s && ./pipe-server.py&" % pipe_server_dir)
-- 
GitLab


From efcaf6fb1a2467e6ec7676267caa967472a394c7 Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 6 Jun 2021 17:20:05 +0200
Subject: [PATCH 16/18] forgot to change jwt_key file name

---
 stage6-core/05-pier-admin/files/config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stage6-core/05-pier-admin/files/config.json b/stage6-core/05-pier-admin/files/config.json
index 1f24d08..e90c561 100644
--- a/stage6-core/05-pier-admin/files/config.json
+++ b/stage6-core/05-pier-admin/files/config.json
@@ -1,6 +1,6 @@
 {
     "log_level": "info",
-    "jwt_key_file": "REPLACE_WITH_SECRETS_DIR/WEBSOCKETD_KEY",
+    "jwt_key_file": "REPLACE_WITH_SECRETS_DIR/PIPE_JWT_KEY",
     "authorized_commands_file": "REPLACE_WITH_ADMIN_DIRECTORY/authorized_commands",
     "cmd_pipe": "REPLACE_WITH_PIPE_DIRECTORY/command_pipe",
     "out_pipe": "REPLACE_WITH_PIPE_DIRECTORY/output_pipe",
-- 
GitLab


From ff28730d4c5ecf6904ccf13b0e37f0ea3b6de7fd Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 6 Jun 2021 20:09:37 +0200
Subject: [PATCH 17/18] strip ansi codes before returning (else breaks pipes
 sometime)

---
 stage6-core/05-pier-admin/files/pipe-server.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/stage6-core/05-pier-admin/files/pipe-server.py b/stage6-core/05-pier-admin/files/pipe-server.py
index 5b78757..ae78d71 100755
--- a/stage6-core/05-pier-admin/files/pipe-server.py
+++ b/stage6-core/05-pier-admin/files/pipe-server.py
@@ -9,13 +9,15 @@ import subprocess
 import logging
 import traceback
 from typing import Dict
-import jwt 
+import jwt
 import pprint
 import os
 import errno
 import tarfile
+import re
 
 pipe_server_dir="REPLACE_WITH_ADMIN_DIRECTORY"
+ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
 
 def usage(): 
     print ("$ cd %s && ./pipe-server.py&" % pipe_server_dir)
@@ -95,7 +97,12 @@ class Pipes:
                 logging.error(oe)
                 raise
     def send_pipe(self, dict: Dict):
-        res = json.dumps(dict, separators=(',', ':'))
+        if dict.get('line'):
+            dict['line'] = ansi_escape.sub('', dict['line'])
+        elif dict.get('out'): 
+            for i in range(len(dict['out'])):
+                dict['out'][i] = ansi_escape.sub('', dict['out'][i]) 
+        res = json.dumps(dict, separators=(',', ':')) 
         # open output pipe for writing
         with open(self.out_pipe, 'w') as out_pipe:
             logging.info("return >{0}<".format(res))
-- 
GitLab


From f5c0b3b3ad138e2714ca0ea1b229156861032aed Mon Sep 17 00:00:00 2001
From: olivier <olivier@levillain.ml>
Date: Sun, 6 Jun 2021 20:14:40 +0200
Subject: [PATCH 18/18] typo

---
 stage6-core/05-pier-admin/files/update-system.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stage6-core/05-pier-admin/files/update-system.sh b/stage6-core/05-pier-admin/files/update-system.sh
index f8aa6f3..e327dd6 100644
--- a/stage6-core/05-pier-admin/files/update-system.sh
+++ b/stage6-core/05-pier-admin/files/update-system.sh
@@ -1,4 +1,4 @@
-#:/bin/bash
+#!/bin/bash
 
 echo "Updating $(hostname) pier host operating system..."
 apt-get update
-- 
GitLab