use python
Nguyen Huu Thuong
5 years ago
parent
124061301d
commit
ecd50df959
@ -1,22 +1,27 @@
|
|||||||
FROM alpine:latest
|
FROM python:3.8.3-slim-buster
|
||||||
|
|
||||||
LABEL "maintainer"="Scott Ng <thuongnht@gmail.com>"
|
LABEL "maintainer"="Scott Ng <thuongnht@gmail.com>"
|
||||||
LABEL "repository"="https://github.com/cross-the-world/ssh-scp-ssh-pipelines"
|
LABEL "repository"="https://github.com/cross-the-world/ssh-scp-ssh-pipelines"
|
||||||
LABEL "version"="1.0.0"
|
LABEL "version"="latest"
|
||||||
|
|
||||||
LABEL "com.github.actions.name"="ssh-scp-ssh-pipelines"
|
LABEL "com.github.actions.name"="ssh-scp-ssh-pipelines"
|
||||||
LABEL "com.github.actions.description"="Pipelines: ssh -> scp -> ssh"
|
LABEL "com.github.actions.description"="Pipeline: ssh -> scp -> ssh"
|
||||||
LABEL "com.github.actions.icon"="terminal"
|
LABEL "com.github.actions.icon"="terminal"
|
||||||
LABEL "com.github.actions.color"="gray-dark"
|
LABEL "com.github.actions.color"="gray-dark"
|
||||||
|
|
||||||
RUN apk update && \
|
RUN apt-get update -y && \
|
||||||
apk add ca-certificates && \
|
apt-get install -y ca-certificates openssh-client openssl sshpass
|
||||||
apk add --no-cache openssh-client openssl openssh sshpass && \
|
|
||||||
apk add --no-cache --upgrade bash openssh sshpass && \
|
|
||||||
rm -rf /var/cache/apk/*
|
|
||||||
|
|
||||||
COPY entrypoint.sh /entrypoint.sh
|
COPY requirements.txt /requirements.txt
|
||||||
COPY test /opt/test
|
RUN pip3 install -r /requirements.txt
|
||||||
RUN chmod +x /entrypoint.sh
|
|
||||||
|
|
||||||
ENTRYPOINT ["/entrypoint.sh"]
|
RUN mkdir -p /opt/tools
|
||||||
|
WORKDIR /opt/tools
|
||||||
|
|
||||||
|
COPY entrypoint.sh /opt/tools/entrypoint.sh
|
||||||
|
RUN chmod +x /opt/tools/entrypoint.sh
|
||||||
|
|
||||||
|
COPY app.py /opt/tools/app.py
|
||||||
|
RUN chmod +x /opt/tools/app.py
|
||||||
|
|
||||||
|
ENTRYPOINT ["./entrypoint.sh"]
|
||||||
@ -0,0 +1,136 @@
|
|||||||
|
from os import environ, path
|
||||||
|
from glob import glob
|
||||||
|
|
||||||
|
import paramiko
|
||||||
|
import scp
|
||||||
|
import sys
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
envs = environ
|
||||||
|
INPUT_HOST = envs.get("INPUT_HOST")
|
||||||
|
INPUT_PORT = int(envs.get("INPUT_PORT", "22"))
|
||||||
|
INPUT_USER = envs.get("INPUT_USER")
|
||||||
|
INPUT_PASS = envs.get("INPUT_PASS")
|
||||||
|
INPUT_KEY = envs.get("INPUT_KEY")
|
||||||
|
INPUT_CONNECT_TIMEOUT = envs.get("INPUT_CONNECT_TIMEOUT", "30s")
|
||||||
|
INPUT_SCP = envs.get("INPUT_SCP")
|
||||||
|
INPUT_FIRST_SSH = envs.get("INPUT_FIRST_SSH")
|
||||||
|
INPUT_LAST_SSH = envs.get("INPUT_LAST_SSH")
|
||||||
|
|
||||||
|
|
||||||
|
seconds_per_unit = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "M": 86400*30}
|
||||||
|
pattern_seconds_per_unit = re.compile(r'^(' + "|".join(['\\d+'+k for k in seconds_per_unit.keys()]) + ')$')
|
||||||
|
|
||||||
|
|
||||||
|
def convert_to_seconds(s):
|
||||||
|
if s is None:
|
||||||
|
return 30
|
||||||
|
if isinstance(s, str):
|
||||||
|
return int(s[:-1]) * seconds_per_unit[s[-1]] if pattern_seconds_per_unit.search(s) else 30
|
||||||
|
if (isinstance(s, int) or isinstance(s, float)) and not math.isnan(s):
|
||||||
|
return round(s)
|
||||||
|
return 30
|
||||||
|
|
||||||
|
|
||||||
|
def connect(callback=None):
|
||||||
|
with paramiko.SSHClient() as ssh:
|
||||||
|
p_key = paramiko.RSAKey.from_private_key(INPUT_KEY) if INPUT_KEY else None
|
||||||
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
|
ssh.connect(INPUT_HOST, port=INPUT_PORT, username=INPUT_USER,
|
||||||
|
pkey=p_key, password=INPUT_PASS,
|
||||||
|
timeout=convert_to_seconds(INPUT_CONNECT_TIMEOUT))
|
||||||
|
if callback:
|
||||||
|
callback(ssh)
|
||||||
|
|
||||||
|
|
||||||
|
# Define progress callback that prints the current percentage completed for the file
|
||||||
|
def progress(filename, size, sent):
|
||||||
|
sys.stdout.write(f"{filename}... {float(sent)/float(size)*100:.2f}%\n")
|
||||||
|
|
||||||
|
|
||||||
|
def ssh_process(ssh, input_ssh):
|
||||||
|
commands = [c.strip() for c in input_ssh.splitlines() if c is not None]
|
||||||
|
command_str = ""
|
||||||
|
l = len(commands)
|
||||||
|
for i in range(len(commands)):
|
||||||
|
c = path.expandvars(commands[i])
|
||||||
|
if c == "":
|
||||||
|
continue
|
||||||
|
if c.endswith('&&') or c.endswith('||') or c.endswith(';'):
|
||||||
|
c = c[0:-2] if i == (l-1) else c
|
||||||
|
else:
|
||||||
|
c = f"{c} &&" if i < (l-1) else c
|
||||||
|
command_str = f"{command_str} {c}"
|
||||||
|
command_str = command_str.strip()
|
||||||
|
print(command_str)
|
||||||
|
|
||||||
|
stdin, stdout, stderr = ssh.exec_command(command_str)
|
||||||
|
|
||||||
|
err = "".join(stderr.readlines())
|
||||||
|
err = err.strip() if err is not None else None
|
||||||
|
if err:
|
||||||
|
raise Exception(f"SSH failed:\n{err}")
|
||||||
|
|
||||||
|
print("".join(stdout.readlines()))
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def scp_process(ssh, input_scp):
|
||||||
|
copy_list = []
|
||||||
|
for c in input_scp.splitlines():
|
||||||
|
if not c:
|
||||||
|
continue
|
||||||
|
l2r = c.split("=>")
|
||||||
|
if len(l2r) == 2:
|
||||||
|
local = l2r[0].strip()
|
||||||
|
remote = l2r[1].strip()
|
||||||
|
if local and remote:
|
||||||
|
copy_list.append({"l": local, "r": remote})
|
||||||
|
continue
|
||||||
|
print(f"SCP ignored {c.strip()}")
|
||||||
|
print(copy_list)
|
||||||
|
|
||||||
|
if len(copy_list) <= 0:
|
||||||
|
print("SCP no copy list found")
|
||||||
|
return
|
||||||
|
|
||||||
|
with scp.SCPClient(ssh.get_transport(), progress=progress, sanitize=lambda x: x) as conn:
|
||||||
|
for l2r in copy_list:
|
||||||
|
remote = l2r.get('r')
|
||||||
|
ssh.exec_command(f"mkdir -p {remote} || true")
|
||||||
|
for f in [f for f in glob(l2r.get('l'))]:
|
||||||
|
conn.put(f, remote_path=remote, recursive=True)
|
||||||
|
print(f"{f} -> {remote}")
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def processes():
|
||||||
|
if INPUT_KEY is None and INPUT_PASS is None:
|
||||||
|
print("SSH-SCP-SSH invalid (Key/Passwd)")
|
||||||
|
return
|
||||||
|
|
||||||
|
if not INPUT_FIRST_SSH:
|
||||||
|
print("SSH-SCP-SSH no first_ssh input found")
|
||||||
|
return
|
||||||
|
print("+++++++++++++++++++Pipeline: RUNNING FIRST SSH+++++++++++++++++++")
|
||||||
|
connect(lambda c: ssh_process(c, INPUT_FIRST_SSH))
|
||||||
|
|
||||||
|
if not INPUT_SCP:
|
||||||
|
print("SSH-SCP-SSH no scp input found")
|
||||||
|
return
|
||||||
|
print("+++++++++++++++++++Pipeline: RUNNING SCP+++++++++++++++++++")
|
||||||
|
connect(lambda c: scp_process(c, INPUT_SCP))
|
||||||
|
|
||||||
|
if not INPUT_LAST_SSH:
|
||||||
|
print("SSH-SCP-SSH no last_ssh input found")
|
||||||
|
return
|
||||||
|
print("+++++++++++++++++++Pipeline: RUNNING LAST SSH+++++++++++++++++++")
|
||||||
|
connect(lambda c: ssh_process(c, INPUT_LAST_SSH))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
processes()
|
||||||
|
|
||||||
|
|
||||||
@ -1,149 +1,7 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
createKeyFile() {
|
|
||||||
local SSH_PATH="$HOME/.ssh"
|
|
||||||
|
|
||||||
mkdir -p "$SSH_PATH"
|
|
||||||
touch "$SSH_PATH/known_hosts"
|
|
||||||
|
|
||||||
echo "$INPUT_KEY" > "$SSH_PATH/id_rsa"
|
|
||||||
|
|
||||||
chmod 700 "$SSH_PATH"
|
|
||||||
chmod 600 "$SSH_PATH/known_hosts"
|
|
||||||
chmod 600 "$SSH_PATH/id_rsa"
|
|
||||||
|
|
||||||
eval $(ssh-agent)
|
|
||||||
ssh-add "$SSH_PATH/id_rsa"
|
|
||||||
|
|
||||||
ssh-keyscan -t rsa "$INPUT_HOST" >> "$SSH_PATH/known_hosts"
|
|
||||||
}
|
|
||||||
|
|
||||||
executeSSH() {
|
|
||||||
local USEPASS=$1
|
|
||||||
local LINES=$2
|
|
||||||
local COMMAND=""
|
|
||||||
|
|
||||||
# holds all commands separated by semi-colon or keep "&&"
|
|
||||||
local COMMANDS=""
|
|
||||||
|
|
||||||
# this while read each commands in line and
|
|
||||||
# evaluate each line against all environment variables
|
|
||||||
while IFS= read -r LINE; do
|
|
||||||
LINE=$(echo $LINE)
|
|
||||||
if [[ -z "${LINE}" ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
COMBINE="&&"
|
|
||||||
LASTCOMBINE=""
|
|
||||||
if [[ $LINE =~ ^.*\&\&$ ]]; then
|
|
||||||
LINE="$LINE true"
|
|
||||||
LASTCOMBINE="&&"
|
|
||||||
elif [[ $LINE =~ ^\&\&.*$ ]]; then
|
|
||||||
LINE="true $LINE"
|
|
||||||
elif [[ $LINE =~ ^.*\|\|$ ]]; then
|
|
||||||
LINE="$LINE false"
|
|
||||||
LASTCOMBINE="||"
|
|
||||||
elif [[ $LINE =~ ^\|\|.*$ ]]; then
|
|
||||||
LINE="false $LINE"
|
|
||||||
COMBINE="||"
|
|
||||||
fi
|
|
||||||
LINE=$(eval 'echo "$LINE"')
|
|
||||||
if ! [[ $LINE =~ ^\(.*\)$ ]]; then
|
|
||||||
LINE=$(eval echo "$LINE")
|
|
||||||
fi
|
|
||||||
LINE="$LINE $LASTCOMBINE"
|
|
||||||
|
|
||||||
if [ -z "$COMMANDS" ]; then
|
|
||||||
COMMANDS="$LINE"
|
|
||||||
else
|
|
||||||
# ref. https://unix.stackexchange.com/questions/459923/multiple-commands-in-sshpass
|
|
||||||
if [[ $COMMANDS =~ ^.*\&\&$ ]] || [[ $COMMANDS =~ ^.*\|\|$ ]]; then
|
|
||||||
COMMANDS="$COMMANDS $LINE"
|
|
||||||
else
|
|
||||||
COMMANDS="$COMMANDS $COMBINE $LINE"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done <<< "$LINES"
|
|
||||||
|
|
||||||
if [[ $COMMANDS =~ ^.*\&\&$ ]]; then
|
|
||||||
COMMANDS="$COMMANDS true"
|
|
||||||
elif [[ $COMMANDS =~ ^.*\|\|$ ]]; then
|
|
||||||
COMMANDS="$COMMANDS false"
|
|
||||||
fi
|
|
||||||
echo "$COMMANDS"
|
|
||||||
|
|
||||||
CMD="ssh"
|
|
||||||
if $USEPASS; then
|
|
||||||
CMD="sshpass -p $INPUT_PASS ssh"
|
|
||||||
fi
|
|
||||||
$CMD -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -p "${INPUT_PORT:-22}" "$INPUT_USER"@"$INPUT_HOST" "$COMMANDS" > /dev/stdout
|
|
||||||
}
|
|
||||||
|
|
||||||
executeSCP() {
|
|
||||||
local USEPASS=$1
|
|
||||||
local LINES=$2
|
|
||||||
local COMMAND=
|
|
||||||
|
|
||||||
CMD="scp"
|
|
||||||
if $USEPASS; then
|
|
||||||
CMD="sshpass -p $INPUT_PASS scp"
|
|
||||||
fi
|
|
||||||
|
|
||||||
while IFS= read -r LINE; do
|
|
||||||
delimiter="=>"
|
|
||||||
LINE=$(echo $LINE)
|
|
||||||
if [[ -z "${LINE}" ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
s=$LINE$delimiter
|
|
||||||
arr=()
|
|
||||||
while [[ $s ]]; do
|
|
||||||
arr+=( "${s%%"$delimiter"*}" );
|
|
||||||
s=${s#*"$delimiter"};
|
|
||||||
done;
|
|
||||||
LOCAL=$(eval 'echo "${arr[0]}"')
|
|
||||||
LOCAL=$(eval echo "$LOCAL")
|
|
||||||
REMOTE=$(eval 'echo "${arr[1]}"')
|
|
||||||
REMOTE=$(eval echo "$REMOTE")
|
|
||||||
|
|
||||||
if [[ -z "${LOCAL}" ]] || [[ -z "${REMOTE}" ]]; then
|
|
||||||
echo "LOCAL/REMOTE can not be parsed $LINE"
|
|
||||||
else
|
|
||||||
echo "Copying $LOCAL ---> $REMOTE"
|
|
||||||
$CMD -r -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -P "${INPUT_PORT:-22}" $LOCAL "$INPUT_USER"@"$INPUT_HOST":$REMOTE > /dev/stdout
|
|
||||||
fi
|
|
||||||
done <<< "$LINES"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
######################################################################################
|
|
||||||
|
|
||||||
echo "+++++++++++++++++++STARTING PIPELINES+++++++++++++++++++"
|
echo "+++++++++++++++++++STARTING PIPELINES+++++++++++++++++++"
|
||||||
|
|
||||||
USEPASS=true
|
python3 /opt/tools/app.py
|
||||||
if [[ -z "${INPUT_KEY}" ]]; then
|
|
||||||
echo "+++++++++++++++++++Use password+++++++++++++++++++"
|
|
||||||
else
|
|
||||||
echo "+++++++++++++++++++Create Key File+++++++++++++++++++"
|
|
||||||
USEPASS=false
|
|
||||||
createKeyFile || false
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! [[ -z "${INPUT_FIRST_SSH}" ]]; then
|
|
||||||
echo "+++++++++++++++++++Step 1: RUNNING SSH+++++++++++++++++++"
|
|
||||||
executeSSH "$USEPASS" "$INPUT_FIRST_SSH" || false
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! [[ -z "${INPUT_SCP}" ]]; then
|
|
||||||
echo "+++++++++++++++++++Step 2: RUNNING SCP+++++++++++++++++++"
|
|
||||||
executeSCP "$USEPASS" "$INPUT_SCP" || false
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! [[ -z "${INPUT_LAST_SSH}" ]]; then
|
|
||||||
echo "+++++++++++++++++++Step 3: RUNNING SSH+++++++++++++++++++"
|
|
||||||
executeSSH "$USEPASS" "$INPUT_LAST_SSH" || false
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "+++++++++++++++++++END PIPELINES+++++++++++++++++++"
|
echo "+++++++++++++++++++END PIPELINES+++++++++++++++++++"
|
||||||
|
|||||||
@ -0,0 +1,2 @@
|
|||||||
|
paramiko
|
||||||
|
scp
|
||||||
Loading…
Reference in New Issue