From de73b968bca4ac137833b4c509fe89bea9e6a1da Mon Sep 17 00:00:00 2001 From: bkraul Date: Wed, 8 Feb 2023 13:46:00 -0600 Subject: [PATCH] Prepare for public release --- docker-children | 51 +++++++++++++++++++++++++++++++++++++++++++ docker-clean-all | 9 ++++++++ docker-destroy-all | 5 +++++ docker-ls | 3 +++ docker-net-optimize | 21 ++++++++++++++++++ docker-node | 17 +++++++++++++++ docker-ps | 3 +++ docker-service-exec | 53 +++++++++++++++++++++++++++++++++++++++++++++ docker-service-ps | 2 ++ docker-stack-deploy | 18 +++++++++++++++ docker-stack-rm | 22 +++++++++++++++++++ docker-start-all | 3 +++ docker-stop-all | 3 +++ docker-system-prune | 12 ++++++++++ docker-update-all | 28 ++++++++++++++++++++++++ 15 files changed, 250 insertions(+) create mode 100755 docker-children create mode 100755 docker-clean-all create mode 100755 docker-destroy-all create mode 100755 docker-ls create mode 100755 docker-net-optimize create mode 100755 docker-node create mode 100755 docker-ps create mode 100755 docker-service-exec create mode 100755 docker-service-ps create mode 100755 docker-stack-deploy create mode 100755 docker-stack-rm create mode 100755 docker-start-all create mode 100755 docker-stop-all create mode 100755 docker-system-prune create mode 100755 docker-update-all diff --git a/docker-children b/docker-children new file mode 100755 index 0000000..391c4aa --- /dev/null +++ b/docker-children @@ -0,0 +1,51 @@ +#!/usr/bin/python3 +# +# usage: python3 docker_descendants.py ... + +import sys +from subprocess import check_output + + +def main(images): + image_ids = set(images) + all_images = docker_images('--all', '--quiet') + all_links = parse_links(docker_links(all_images)) + descendants = desc(image_ids, all_links) + pred = lambda s: lambda line: s[:12] in line + match = list(map(pred, descendants)) + return filter(lambda i: any(s(i) for s in match), docker_images()) + + +def parse_links(lines): + parseid = lambda s: s.replace('sha256:', '') + for line in reversed(list(lines)): + yield list(map(parseid, line.split())) + + +def desc(image_ids, links): + if links: + link, *tail = links + if len(link) > 1: + image_id, parent_id = link + checkid = lambda i: parent_id.startswith(i) + if any(map(checkid, image_ids)): + return desc(image_ids | {image_id}, tail) + return desc(image_ids, tail) + return image_ids + + +def docker_links(images): + cmd = [ 'docker', 'inspect', '--format={{.Id}} {{.Parent}}'] + return run(cmd + images) + + +def docker_images(*args): + return run(('docker', 'images') + args) + + +def run(cmd): + return check_output(cmd, universal_newlines=True).splitlines() + + +if __name__ == '__main__': + print('\n'.join(main(sys.argv[1:]))) diff --git a/docker-clean-all b/docker-clean-all new file mode 100755 index 0000000..bf6901f --- /dev/null +++ b/docker-clean-all @@ -0,0 +1,9 @@ +#!/bin/bash + +# remove exited containers: +docker ps --filter status=dead --filter status=exited -aq | xargs -r docker rm -v +# remove unused images +docker images --no-trunc | grep '' | awk '{ print $3 }' | xargs -r docker rmi -f +# remove unused volumes: +docker volume prune -f + diff --git a/docker-destroy-all b/docker-destroy-all new file mode 100755 index 0000000..0d2a415 --- /dev/null +++ b/docker-destroy-all @@ -0,0 +1,5 @@ +#!/bin/sh + +CONTAINER=$(docker ps -a -q) +docker stop $CONTAINER +docker rm $CONTAINER diff --git a/docker-ls b/docker-ls new file mode 100755 index 0000000..63105ff --- /dev/null +++ b/docker-ls @@ -0,0 +1,3 @@ +#!/bin/bash + +docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}\t{{.Status}}" diff --git a/docker-net-optimize b/docker-net-optimize new file mode 100755 index 0000000..f6a3a1b --- /dev/null +++ b/docker-net-optimize @@ -0,0 +1,21 @@ +#!/bin/bash + +sysctl -w net.core.netdev_max_backlog="150000" +sysctl -w net.core.rmem_max="16777216" +sysctl -w net.core.somaxconn="65535" +sysctl -w net.core.wmem_max="16777216" +sysctl -w net.ipv4.ip_local_port_range="1025 65535" +sysctl -w net.ipv4.tcp_fin_timeout="20" +sysctl -w net.ipv4.tcp_keepalive_time="30" +sysctl -w net.ipv4.tcp_max_syn_backlog="20480" +sysctl -w net.ipv4.tcp_max_tw_buckets="400000" +sysctl -w net.ipv4.tcp_no_metrics_save="1" +sysctl -w net.ipv4.tcp_syn_retries="2" +sysctl -w net.ipv4.tcp_synack_retries="2" +sysctl -w net.ipv4.tcp_tw_reuse="1" +sysctl -w net.ipv4.tcp_timestamps="0" +sysctl -w vm.min_free_kbytes="65536" +sysctl -w vm.overcommit_memory="1" +sysctl -w vm.max_map_count="2621440" +sysctl -w vm.swappiness="10" +sysctl -w vm.dirty_ratio="40" diff --git a/docker-node b/docker-node new file mode 100755 index 0000000..105a200 --- /dev/null +++ b/docker-node @@ -0,0 +1,17 @@ +#!/bin/bash +# this command helper depends on key-based (passwordless) SSH access by the current user to each of the docker swarm nodes. +NODE_HOST=$1; shift; + +nc -zvw3 $NODE_HOST 22 &> /dev/null; +PING_RESULT=$? + +if [ $PING_RESULT != 0 ]; then + echo + echo "Host '$NODE_HOST' not reachable." + exit 2 +fi + +# connect to the docker host through SSH +export DOCKER_HOST="ssh://$(whoami)@$NODE_HOST.$(hostname -d)" +# execute the command +docker $@ \ No newline at end of file diff --git a/docker-ps b/docker-ps new file mode 100755 index 0000000..234be85 --- /dev/null +++ b/docker-ps @@ -0,0 +1,3 @@ +#!/bin/bash + +docker stats --no-stream --format "table {{.Name}}\t{{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}" | sort -k 4 -h diff --git a/docker-service-exec b/docker-service-exec new file mode 100755 index 0000000..ad8e497 --- /dev/null +++ b/docker-service-exec @@ -0,0 +1,53 @@ +#!/bin/bash +# this command helper depends on key-based (passwordless) SSH access by the current user to each of the docker swarm nodes. +# usage help. +usage() +{ + echo + echo "Usage: docker-service-exec [OPTIONS] SERVICE COMMAND [ARG...]" + echo + echo "Run a command in a running swarm service" + echo + echo "Options: + -i, --interactive Keep STDIN open even if not attached + -u, --user string Username or UID (format: [:]) + -w, --workdir string Working directory inside the container" + exit 2 +} + +# parse the arguments +PARSED_ARGUMENTS=$(getopt -q -a -n docker-service-exec -o iu:w: -l interactive,it,user:,workdir: -- "$@") +VALID_ARGUMENTS=$? +if [ "$VALID_ARGUMENTS" != "0" ]; then + usage +fi + +# use below for diagnostics only. +# echo "PARSED_ARGUMENTS is $PARSED_ARGUMENTS" +eval set -- "$PARSED_ARGUMENTS" +while : +do + case "$1" in + -i | --it | --interactive) OPT_INTERACTIVE="-it"; shift ;; + -u | --user) OPT_USER="-u $2"; shift 2 ;; + -w | --workdir) OPT_WORKDIR="-w $2"; shift 2 ;; + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + # If invalid options were passed, then getopt should have reported an error, + # which we checked as VALID_ARGUMENTS when getopt was called... + *) + echo "Unexpected option: $1 - this should not happen." + usage;; + esac +done + +# discover docker server vars. +SERVICE_NAME=$1; shift +TASK_ID=$(docker service ps --filter 'desired-state=running' $SERVICE_NAME -q) +NODE_ID=$(docker inspect --format '{{ .NodeID }}' $TASK_ID) +CONTAINER_ID=$(docker inspect --format '{{ .Status.ContainerStatus.ContainerID }}' $TASK_ID) +NODE_HOST=$(docker node inspect --format '{{ .Description.Hostname }}' $NODE_ID) +# connect to the docker host through SSH +export DOCKER_HOST="ssh://$(whoami)@$NODE_HOST.$(hostname -d)" +# execute the command +docker exec $OPT_INTERACTIVE $OPT_USER $OPT_WORKDIR $CONTAINER_ID "$@" \ No newline at end of file diff --git a/docker-service-ps b/docker-service-ps new file mode 100755 index 0000000..d545b42 --- /dev/null +++ b/docker-service-ps @@ -0,0 +1,2 @@ +#!/bin/bash +docker node ps $(docker node ls -q) --filter 'desired-state=Running' --format "table {{ .ID }}\t{{ .Name }}\t{{ .Image }}\t{{ .Node }}\t{{ .DesiredState }}\t{{ .CurrentState }}" | uniq \ No newline at end of file diff --git a/docker-stack-deploy b/docker-stack-deploy new file mode 100755 index 0000000..845b470 --- /dev/null +++ b/docker-stack-deploy @@ -0,0 +1,18 @@ +#!/bin/bash + +# assumes compose files for swarm are specified as docker-compose.stack.yml. + +# identify the parent directory in order to name the service. +PARENT_DIR="${PWD##*/}" +# replace restricted characters. +PARENT_DIR=$( echo $PARENT_DIR | tr ' '. _ ) + +# attempt to find a suitable compose file. +if [ ! -f "docker-compose.stack.yml" ]; then + echo "No suitable stack compose file found in directory."; + exit 1; +fi + +# if we found a compose file, deploy it. +echo "Deploying stack as ${PARENT_DIR}..." +docker stack deploy -c docker-compose.stack.yml --with-registry-auth ${PARENT_DIR} \ No newline at end of file diff --git a/docker-stack-rm b/docker-stack-rm new file mode 100755 index 0000000..07ed246 --- /dev/null +++ b/docker-stack-rm @@ -0,0 +1,22 @@ +#!/bin/bash + +# identify the parent directory in order to name the service. +PARENT_DIR="${PWD##*/}" +# replace restricted characters. +PARENT_DIR=$( echo $PARENT_DIR | tr ' '. _ ) +# retrieve any stack matching the parent dir. +RUNNING_STACK=$( docker stack ls | grep $PARENT_DIR ) + +# attempt to find a running stack with the specified name. +if [ -z "$RUNNING_STACK" ]; then + echo "No running stacks were found matching '$PARENT_DIR'."; + exit 1; +fi +# sanitize the running stack string. +RUNNING_STACK=$( echo $PARENT_DIR | cut -d " " -f1 ) + +# if we found a running stack, bring it down. +echo "Removing stack $RUNNING_STACK..." +docker stack rm $RUNNING_STACK +echo "Waiting for 30 seconds for services to terminate..." +sleep 30 \ No newline at end of file diff --git a/docker-start-all b/docker-start-all new file mode 100755 index 0000000..4239418 --- /dev/null +++ b/docker-start-all @@ -0,0 +1,3 @@ +#!/bin/sh + +docker start $(docker ps -a -q) diff --git a/docker-stop-all b/docker-stop-all new file mode 100755 index 0000000..912b23b --- /dev/null +++ b/docker-stop-all @@ -0,0 +1,3 @@ +#!/bin/sh + +docker stop $(docker ps -a -q) diff --git a/docker-system-prune b/docker-system-prune new file mode 100755 index 0000000..4b7f5ce --- /dev/null +++ b/docker-system-prune @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +#set -ex + +SSH_USER=$(whoami) +SWARM_MANAGER_IP=$(docker node inspect self --format '{{ .Status.Addr }}') +HOSTS=$(docker node ls --format="{{ .Hostname }}") +echo +for HOSTNAME in ${HOSTS}; do + echo "Processing system prune operation for host: ${HOSTNAME}..." + ssh "${SSH_USER}@${HOSTNAME}.$(hostname -d)" 'docker system prune -f'; + echo +done \ No newline at end of file diff --git a/docker-update-all b/docker-update-all new file mode 100755 index 0000000..6fe258d --- /dev/null +++ b/docker-update-all @@ -0,0 +1,28 @@ +#!/bin/bash + +if [ ! -z $1 ]; then + START_DIR="$1" +else + START_DIR="$HOME/docker-compose" +fi + +if [ ! -d "$START_DIR" ]; then + echo "Invalid directory '$START_DIR': Exiting." + exit 1 +fi + +echo "Current dir is $START_DIR" + +for f in $(find $START_DIR/ -iname "docker-compose.yml") +do + CURRENT_DIR=$(dirname "${f}") + echo "Updating $CURRENT_DIR..." + cd $CURRENT_DIR + if [ ! -f ".update.ignore" ]; then + docker-compose pull --no-parallel && docker-compose up -d --remove-orphans 2>&1 + else + echo "Stack flagged for ignore updates" + fi +done + +cd $START_DIR