From 97cd91a020ca418dc686eabcb35ab5b7aabf83ab Mon Sep 17 00:00:00 2001 From: "aynic.os" Date: Mon, 14 Jun 2021 12:52:48 +0200 Subject: [PATCH] wip --- .env.dist | 1 - Makefile | 2 +- ansible/ansible.mk | 20 +- ansible/def.ansible.mk | 23 +- ansible/roles/hosts/files/etc/profile.d/rc.sh | 10 +- .../hosts/files/etc/profile.d/rc_functions.sh | 385 ++++++++++++--------- ansible/roles/hosts/templates/Makefile.j2 | 12 +- aws/aws.mk | 10 +- aws/def.aws.mk | 22 +- docker/ssh/Dockerfile | 5 +- docker/theia/rc_functions.sh | 375 ++++++++++++++------ make/apps/build.mk | 4 +- make/apps/common.mk | 12 +- make/apps/def.build.mk | 3 +- make/apps/def.docker.mk | 58 ++-- make/apps/def.mk | 6 +- make/apps/docker.mk | 24 +- make/apps/git.mk | 70 ++-- make/apps/install.mk | 26 +- make/apps/myos/def.ssh.mk | 4 +- make/apps/myos/ssh.mk | 53 ++- make/apps/release.mk | 2 +- make/apps/setup.mk | 6 +- make/common.mk | 10 +- make/def.docker.mk | 69 ++-- make/def.mk | 57 ++- make/env.mk | 8 +- make/help.mk | 29 +- make/monorepo/common.mk | 3 +- make/monorepo/release.mk | 2 +- make/monorepo/subrepo.mk | 34 +- make/utils.mk | 3 +- openstack/def.openstack.mk | 10 +- packer/def.packer.mk | 29 +- stack/base.mk | 24 +- stack/base/base.yml | 15 - stack/node.mk | 12 +- stack/sematext/logagent.yml | 4 - stack/theia/theia.yml | 9 - terraform/def.terraform.mk | 14 +- 40 files changed, 830 insertions(+), 635 deletions(-) diff --git a/.env.dist b/.env.dist index e1ad631..58bf4aa 100644 --- a/.env.dist +++ b/.env.dist @@ -1,2 +1 @@ -APP=myos DOCKER_SERVICE=cli diff --git a/Makefile b/Makefile index 95f2c74..bdbd76c 100644 --- a/Makefile +++ b/Makefile @@ -12,4 +12,4 @@ app-build: base install-build-config app-install: base node -app-start: base-ssh-add +app-start: ssh-add diff --git a/ansible/ansible.mk b/ansible/ansible.mk index 5069d90..257f355 100644 --- a/ansible/ansible.mk +++ b/ansible/ansible.mk @@ -1,15 +1,11 @@ # target ansible: Fire docker-build-ansible, Call ansible ANSIBLE_ARGS ARGS or ansible-run target .PHONY: ansible -ansible: docker-build-ansible -ifneq ($(ARGS),) +ansible: $(if $(DOCKER_RUN),docker-build-ansible) $(call ansible,$(ANSIBLE_ARGS) $(ARGS)) -else - $(call make,ansible-run) -endif # target ansible-playbook: Call ansible-playbook ANSIBLE_ARGS ARGS .PHONY: ansible-playbook -ansible-playbook: +ansible-playbook: $(if $(DOCKER_RUN),docker-build-ansible) $(call ansible-playbook,$(ANSIBLE_ARGS) $(ARGS)) # target ansible-pull: Call ansible-pull ANSIBLE_GIT_REPOSITORY ANSIBLE_PLAYBOOK @@ -17,16 +13,16 @@ ansible-playbook: ansible-pull: $(call ansible-pull,--url $(ANSIBLE_GIT_REPOSITORY) $(if $(ANSIBLE_GIT_KEY_FILE),--key-file $(ANSIBLE_GIT_KEY_FILE)) $(if $(ANSIBLE_GIT_VERSION),--checkout $(ANSIBLE_GIT_VERSION)) $(if $(ANSIBLE_GIT_DIRECTORY),--directory $(ANSIBLE_GIT_DIRECTORY)) $(if $(ANSIBLE_TAGS),--tags $(ANSIBLE_TAGS)) $(if $(ANSIBLE_EXTRA_VARS),--extra-vars '$(ANSIBLE_EXTRA_VARS)') $(if $(findstring true,$(FORCE)),--force) $(if $(findstring true,$(DRYRUN)),--check) --full $(if $(ANSIBLE_INVENTORY),--inventory $(ANSIBLE_INVENTORY)) $(ANSIBLE_PLAYBOOK)) -# target ansible-pull@%: Fire get-PrivateIpAddress-% for SERVER_NAME, Call ssh-exec make ansible-pull DOCKER_IMAGE_TAG +# target ansible-pull@%: Fire ssh-get-PrivateIpAddress-% for SERVER_NAME, Call ssh-exec make ansible-pull DOCKER_IMAGE_TAG .PHONY: ansible-pull@% -ansible-pull@%: get-PrivateIpAddress-$(SERVER_NAME) +ansible-pull@%: ssh-get-PrivateIpAddress-$(SERVER_NAME) $(call ssh-exec,$(AWS_INSTANCE_IP),make ansible-pull ANSIBLE_DOCKER_IMAGE_TAG=$(ANSIBLE_DOCKER_IMAGE_TAG) ANSIBLE_TAGS=$(ANSIBLE_TAGS) FORCE=$(FORCE)) -# target ansible-run: Fire base-ssh-add docker-build-ansible ansible-run-localhost +# target ansible-run: Fire ssh-add ansible-run-localhost .PHONY: ansible-run -ansible-run: base-ssh-add docker-build-ansible ansible-run-localhost +ansible-run: ssh-add ansible-run-localhost -# target ansible-run-%: Call ansible-playbook ANSIBLE_PLAYBOOK +# target ansible-run-%: Fire docker-build-ansible, Call ansible-playbook ANSIBLE_PLAYBOOK .PHONY: ansible-run-% -ansible-run-%: +ansible-run-%: $(if $(DOCKER_RUN),docker-build-ansible) $(call ansible-playbook,$(if $(ANSIBLE_TAGS),--tags $(ANSIBLE_TAGS)) $(if $(ANSIBLE_EXTRA_VARS),--extra-vars '$(patsubst target=localhost,target=$*,$(ANSIBLE_EXTRA_VARS))') $(if $(findstring true,$(DRYRUN)),--check) $(if $(ANSIBLE_INVENTORY),--inventory $(ANSIBLE_INVENTORY)) $(ANSIBLE_PLAYBOOK)) diff --git a/ansible/def.ansible.mk b/ansible/def.ansible.mk index 0c7a986..6b935a1 100644 --- a/ansible/def.ansible.mk +++ b/ansible/def.ansible.mk @@ -23,34 +23,19 @@ CMDS += ansible ansible-playbook DOCKER_RUN_OPTIONS += --add-host=host.docker.internal:$(DOCKER_INTERNAL_DOCKER_HOST) ENV_VARS += ANSIBLE_AWS_ACCESS_KEY_ID ANSIBLE_AWS_DEFAULT_OUTPUT ANSIBLE_AWS_DEFAULT_REGION ANSIBLE_AWS_SECRET_ACCESS_KEY ANSIBLE_CONFIG ANSIBLE_DISKS_NFS_DISK ANSIBLE_DISKS_NFS_OPTIONS ANSIBLE_DISKS_NFS_PATH ANSIBLE_DOCKER_IMAGE_TAG ANSIBLE_DOCKER_REGISTRY ANSIBLE_EXTRA_VARS ANSIBLE_GIT_DIRECTORY ANSIBLE_GIT_KEY_FILE ANSIBLE_GIT_REPOSITORY ANSIBLE_GIT_VERSION ANSIBLE_INVENTORY ANSIBLE_PLAYBOOK ANSIBLE_SSH_PRIVATE_KEYS ANSIBLE_USERNAME ANSIBLE_VERBOSE -ifeq ($(DOCKER), true) -define ansible - $(call INFO,ansible,$(1)) - $(call run,$(DOCKER_REPOSITORY)/ansible:$(DOCKER_IMAGE_TAG) $(ANSIBLE_ARGS) -i $(ANSIBLE_INVENTORY)/.host.docker.internal $(ANSIBLE_VERBOSE) $(1)) -endef -define ansible-playbook - $(call INFO,ansible-playbook,$(1)) - $(call run,--entrypoint=ansible-playbook $(DOCKER_REPOSITORY)/ansible:$(DOCKER_IMAGE_TAG) $(ANSIBLE_ARGS) -i $(ANSIBLE_INVENTORY)/.host.docker.internal $(ANSIBLE_VERBOSE) $(1)) -endef -define ansible-pull - $(call INFO,ansible-pull,$(1)) - # TODO : run ansible in docker and target localhost outside docker - $(call env-exec,ansible-pull $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) -endef -else # function ansible: Call run ansible ANSIBLE_ARGS with arg 1 define ansible $(call INFO,ansible,$(1)) - $(call run,ansible $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) + $(RUN) $(call run,ansible $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(if $(DOCKER_RUN),-i $(ANSIBLE_INVENTORY)/.host.docker.internal) $(1),$(DOCKER_REPOSITORY)/) endef # function ansible-playbook: Call run ansible-playbook ANSIBLE_ARGS with arg 1 define ansible-playbook $(call INFO,ansible-playbook,$(1)) - $(call run,ansible-playbook $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) + $(RUN) $(call run,ansible$(if $(DOCKER_RUN),,-playbook) $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(if $(DOCKER_RUN),-i $(ANSIBLE_INVENTORY)/.host.docker.internal) $(1),--entrypoint=ansible-playbook $(DOCKER_REPOSITORY)/) endef # function ansible-pull: Call run ansible-pull ANSIBLE_ARGS with arg 1 +## TODO: run ansible in docker and target localhost outside docker when DOCKER=true define ansible-pull $(call INFO,ansible-pull,$(1)) - $(call run,ansible-pull $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) + $(RUN) $(call env-run,ansible-pull $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) endef -endif diff --git a/ansible/roles/hosts/files/etc/profile.d/rc.sh b/ansible/roles/hosts/files/etc/profile.d/rc.sh index fe2b69e..6851226 100644 --- a/ansible/roles/hosts/files/etc/profile.d/rc.sh +++ b/ansible/roles/hosts/files/etc/profile.d/rc.sh @@ -1,9 +1,13 @@ # shellcheck shell=sh -# test current shell flags +## rc.sh calls user defined functions +# author: Yann "aya" Autissier +# license: MIT +# updated: 2021/03/04 + case $- in # if we are in an interactive shell *i*) - # load user stuff from files ~/.rc.d/* + # load user stuff from ~/.rc.d/* files for file in "${HOME}"/.rc.d/*; do # read files only if [ -f "${file}" ]; then @@ -19,7 +23,7 @@ case $- in command -v "${func_name}" >/dev/null 2>&1 && "${func_name}" "${func_args}" fi done - # load user stuff from env vars RC_* + # load user stuff from RC_* env vars IFS="$(printf '%b_' '\n')"; IFS="${IFS%_}"; for line in $(printenv 2>/dev/null |awk '$0 ~ /^RC_[1-9A-Z_]*=/'); do func_name=$(printf '%s\n' "${line%%=*}" |awk '{print tolower(substr($0,4))}') eval func_args=\$"${line%%=*}" diff --git a/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh b/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh index ff6d7ba..8ab8678 100644 --- a/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh +++ b/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh @@ -1,218 +1,283 @@ # shellcheck shell=sh +## rc_function.sh defines customs shell functions +# author: Yann "aya" Autissier +# license: MIT +# updated: 2021/03/04 ## force() runs a command sine die force() { - if [ $# -gt 0 ]; then - while true; do - "$@" - sleep 1 - done - fi + if [ $# -gt 0 ]; then + while true; do + "$@" + sleep 1 + done + fi } ## force8() runs a command sine die if not already running force8() { - if [ $# -gt 0 ]; then - while true; do - # awk expression to match $@ - [ "$(ps wwx -o args 2>/dev/null |awk -v field="${PS_X_FIELD:-1}" ' - BEGIN {nargs=split("'"$*"'",args)} - # if match first field - $field == args[1] { - matched=1; - # match following fields - for (i=1;i<=NF-field;i++) { - if ($(i+field) == args[i+1]) {matched++} - } - # all fields matched - if (matched == nargs) {found++} - } - END {print found+0}' - )" = 0 ] && "$@" - sleep 1 - done - fi + if [ $# -gt 0 ]; then + while true; do + # awk expression to match $@ + [ "$(ps wwx -o args 2>/dev/null |awk -v field="${PS_X_FIELD:-1}" ' + BEGIN { nargs=split("'"$*"'",args); } + # first field matched + $field == args[1] { + matched=1; + # match following fields + for (i=1;i<=NF-field;i++) { + if ($(i+field) == args[i+1]) { matched++; } + }; + # all fields matched + if (matched == nargs) { found++; } + } + END { print found+0; }' + )" = 0 ] && "$@" + sleep 1 + done + fi } ## load_average() prints the current load average load_average() { - awk '{printf "%.1f\n" $1}' /proc/loadavg 2>/dev/null \ - || uptime 2>/dev/null |awk '{printf "%.1f\n", $(NF-2)}' -} + awk '{printf "%.1f\n" $1}' /proc/loadavg 2>/dev/null\ + || uptime 2>/dev/null |awk '{printf "%.1f\n", $(NF-2)}' + } ## process_count() prints number of "processes"/"running processes"/"D-state" process_count() { - ps ax -o stat 2>/dev/null |awk ' - $1 ~ /R/ {process_running++}; - $1 ~ /D/ {process_dstate++}; - END { - print NR-1"/"process_running+0"/"process_dstate+0; - }' + ps ax -o stat 2>/dev/null |awk ' + $1 ~ /R/ {process_running++}; + $1 ~ /D/ {process_dstate++}; + END { print NR-1"/"process_running+0"/"process_dstate+0; }' } ## prompt_set() exports custom PROMPT_COMMAND prompt_set() { - case "${TERM}" in - screen*) - ESCAPE_CODE_DCS="\033k" - ESCAPE_CODE_ST="\033\\" - ;; - linux*|xterm*|rxvt*) - ESCAPE_CODE_DCS="\033]0;" - ESCAPE_CODE_ST="\007" - ;; - *) - ;; - esac - # in a screen - if [ -n "${STY}" ]; then - export PROMPT_COMMAND='printf "${ESCAPE_CODE_DCS:-\033]0;}%s${ESCAPE_CODE_ST:-\007}" "${PWD##*/}"' - else - export PROMPT_COMMAND='printf "${ESCAPE_CODE_DCS:-\033]0;}%s@%s:%s${ESCAPE_CODE_ST:-\007}" "${USER}" "${HOSTNAME%%.*}" "${PWD##*/}"' - fi - unset ESCAPE_CODE_DCS ESCAPE_CODE_ST + case "${TERM}" in + screen*) + ESCAPE_CODE_DCS="\033k" + ESCAPE_CODE_ST="\033\\" + ;; + linux*|xterm*|rxvt*) + ESCAPE_CODE_DCS="\033]0;" + ESCAPE_CODE_ST="\007" + ;; + *) + ;; + esac + # in a screen + if [ -n "${STY}" ]; then + export PROMPT_COMMAND='printf\ + "${ESCAPE_CODE_DCS:-\033]0;}%s${ESCAPE_CODE_ST:-\007}"\ + "${PWD##*/}"' + else + export PROMPT_COMMAND='printf\ + "${ESCAPE_CODE_DCS:-\033]0;}%s@%s:%s${ESCAPE_CODE_ST:-\007}"\ + "${USER}"\ + "${HOSTNAME%%.*}"\ + "${PWD##*/}"' + fi + unset ESCAPE_CODE_DCS ESCAPE_CODE_ST } ## ps1_set() exports custom PS1 ps1_set() { - case "$0" in - *sh) - COLOR_DGRAY="\[\033[1;30m\]" - COLOR_RED="\[\033[01;31m\]" - COLOR_GREEN="\[\033[01;32m\]" - COLOR_BROWN="\[\033[0;33m\]" - COLOR_YELLOW="\[\033[01;33m\]" - COLOR_BLUE="\[\033[01;34m\]" - COLOR_CYAN="\[\033[0;36m\]" - COLOR_GRAY="\[\033[0;37m\]" - COLOR_NC="\[\033[0m\]" - ;; + case "$0" in + *sh) + COLOR_DGRAY="\[\033[1;30m\]" + COLOR_RED="\[\033[01;31m\]" + COLOR_GREEN="\[\033[01;32m\]" + COLOR_BROWN="\[\033[0;33m\]" + COLOR_YELLOW="\[\033[01;33m\]" + COLOR_BLUE="\[\033[01;34m\]" + COLOR_CYAN="\[\033[0;36m\]" + COLOR_GRAY="\[\033[0;37m\]" + COLOR_RESET="\[\033[0m\]" + ;; + *) + ;; + esac + + PS1_STATUS="\$?" + PS1_COUNT="${COLOR_DGRAY}[\` + case \"$PS1_STATUS\" in + 0) + printf \"${COLOR_BLUE}${PS1_STATUS}\";; + 1) + printf \"${COLOR_YELLOW}${PS1_STATUS}\";; *) - ;; + printf \"${COLOR_RED}${PS1_STATUS}\";; esac - - PS1_COUNT="${COLOR_DGRAY}[${COLOR_BLUE}\$(process_count 2>/dev/null)${COLOR_DGRAY}|${COLOR_BLUE}\$(user_count 2>/dev/null)${COLOR_DGRAY}|${COLOR_BLUE}\$(load_average 2>/dev/null)${COLOR_DGRAY}]${COLOR_NC}" - PS1_END="${COLOR_DGRAY}\$(if [ \"\$(id -u)\" = 0 ]; then printf \"#\"; else printf \"\$\"; fi)${COLOR_NC}" - if type __git_ps1 >/dev/null 2>&1; then - PS1_GIT="\$(__git_ps1 2>/dev/null \" (%s)\")" + type process_count >/dev/null 2>&1 && printf\ + \"${COLOR_DGRAY}|${COLOR_BLUE}%s\"\ + \"\$(process_count 2>/dev/null)\" + type user_count >/dev/null 2>&1 && printf\ + \"${PS1_COUNT}${COLOR_DGRAY}|${COLOR_BLUE}%s\"\ + \"\$(user_count 2>/dev/null)\" + type load_average >/dev/null 2>&1 && printf\ + \"${PS1_COUNT}${COLOR_DGRAY}|${COLOR_BLUE}%s\"\ + \"\$(load_average 2>/dev/null)\" + \`${COLOR_DGRAY}]${COLOR_RESET}" + PS1_END="${COLOR_DGRAY}\$( + if [ \"\$(id -u)\" = 0 ]; then + printf \"#\"; else - PS1_GIT="\$(BRANCH=\$(git rev-parse --abbrev-ref HEAD 2>/dev/null); [ -n \"\${BRANCH}\" ] && printf \" (\${BRANCH})\")" + printf \"\$\"; fi - PS1_GIT="${COLOR_CYAN}${PS1_GIT}${COLOR_NC}" - PS1_HOSTNAME_COLOR="\`case \"\${ENV}\" in [Pp][Rr][0Oo][Dd]*) printf \"${COLOR_RED}\";; *) if [ -n \"\${ENV}\" ]; then printf \"${COLOR_YELLOW}\"; else printf \"${COLOR_GREEN}\"; fi;; esac\`" - PS1_HOSTNAME="${PS1_HOSTNAME_COLOR}\$(hostname |sed 's/\..*//')${COLOR_NC}" - PS1_USER_COLOR="\$(if [ \"\$(id -u)\" = 0 ]; then printf \"${COLOR_RED}\"; else printf \"${COLOR_BROWN}\"; fi)" - PS1_USER="${PS1_USER_COLOR}\$(id -nu)${COLOR_NC}" - PS1_WORKDIR="${COLOR_GRAY}\$(pwd |sed 's|^'\${HOME}'\(/.*\)*$|~\1|')${COLOR_NC}" - - export PS1="${PS1_COUNT}${PS1_USER}${COLOR_DGRAY}@${PS1_HOSTNAME}${COLOR_DGRAY}:${PS1_WORKDIR}${PS1_GIT}${PS1_END} " - unset PS1_COUNT PS1_END PS1_GIT PS1_HOSTNAME PS1_HOSTNAME_COLOR PS1_USER PS1_USER_COLOR PS1_WORKDIR + )${COLOR_RESET}" + PS1_GIT="\$( + if type __git_ps1 >/dev/null 2>&1; then + printf \"\$(__git_ps1 2>/dev/null \" (%s)\")\" + else + printf \"\$(BRANCH=\$(git rev-parse --abbrev-ref HEAD 2>/dev/null);\ + [ -n \"\${BRANCH}\" ] && printf \" (\${BRANCH})\")\" + fi + )" + PS1_GIT="${COLOR_CYAN}${PS1_GIT}${COLOR_RESET}" + PS1_HOSTNAME_COLOR="\`case \"\${ENV}\" in + [Pp][Rr][0Oo][Dd]*) + printf \"${COLOR_RED}\";; + *) + if [ -n \"\${ENV}\" ]; then + printf \"${COLOR_YELLOW}\"; + else + printf \"${COLOR_GREEN}\"; + fi;; + esac\`" + PS1_HOSTNAME="${PS1_HOSTNAME_COLOR}\$(hostname |sed 's/\..*//')${COLOR_RESET}" + PS1_USER_COLOR="\$( + if [ \"\$(id -u)\" = 0 ]; then + printf \"${COLOR_RED}\"; + else + printf \"${COLOR_BROWN}\"; + fi + )" + PS1_USER="${PS1_USER_COLOR}\$(id -nu):\$(id -u)${COLOR_RESET}" + PS1_WORKDIR="${COLOR_GRAY}\$( + pwd |sed 's|^'\${HOME}'\(/.*\)*$|~\1|' + )${COLOR_RESET}" + PS1="${PS1_COUNT}${PS1_USER}${COLOR_DGRAY}@${PS1_HOSTNAME}" + PS1="${PS1}${COLOR_DGRAY}:${PS1_WORKDIR}${PS1_GIT}${PS1_END} " + export 'PS1' + unset PS1_COUNT PS1_END PS1_GIT PS1_HOSTNAME PS1_HOSTNAME_COLOR\ + PS1_USER PS1_USER_COLOR PS1_STATUS PS1_WORKDIR } ## screen_attach() attaches existing screen session or creates a new one screen_attach() { - command -v screen >/dev/null 2>&1 || return - SCREEN_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" - if [ -z "${STY}" ]; then - # attach screen in tmux window 0 only ;) - [ -n "${TMUX}" ] && [ "$(tmux list-window 2>/dev/null |awk '$NF == "(active)" {print $1}' |sed 's/:$//')" != "0" ] && return - printf 'Attaching screen.' && sleep 1 && printf '.' && sleep 1 && printf '.' && sleep 1 - exec screen -xRR -S "${SCREEN_SESSION}" - fi - unset SCREEN_SESSION + command -v screen >/dev/null 2>&1 || return + SCREEN_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" + if [ -z "${STY}" ]; then + # attach screen in tmux window 0 only ;) + [ -n "${TMUX}" ] \ + && [ "$(tmux list-window 2>/dev/null |awk '$NF == "(active)" {print $1}'\ + |sed 's/:$//')" != "0" ] \ + && return + printf 'Attaching screen.' && sleep 1\ + && printf '.' && sleep 1\ + && printf '.' && sleep 1 + exec screen -xRR -S "${SCREEN_SESSION}" + fi + unset SCREEN_SESSION } ## screen_detach() detaches current screen session screen_detach() { - screen -d + screen -d } - ## ssh_add() loads all private keys in ~/.ssh/ to ssh agent ssh_add() { - command -v ssh-agent >/dev/null 2>&1 && command -v ssh-add >/dev/null 2>&1 || return - SSH_AGENT_DIR="/tmp/ssh-$(id -u)" - SSH_AGENT_SOCK="${SSH_AGENT_DIR}/agent@$(hostname |sed 's/\..*//')" - # launch a new agent - if [ -z "${SSH_AUTH_SOCK}" ]; then - [ ! -d "${SSH_AGENT_DIR}" ] && mkdir -p "${SSH_AGENT_DIR}" 2>/dev/null && chmod 0700 "${SSH_AGENT_DIR}" - # search for an already running agent - if ps wwx -o args |awk '$1 ~ "ssh-agent$" && $3 == "'"${SSH_AGENT_SOCK}"'"' |wc -l |grep -q 0; then - rm -f "${SSH_AGENT_SOCK}" - ssh-agent -a "${SSH_AGENT_SOCK}" >/dev/null 2>&1 - fi + command -v ssh-agent >/dev/null 2>&1 && command -v ssh-add >/dev/null 2>&1 || return + SSH_AGENT_DIR="/tmp/ssh-$(id -u)" + SSH_AGENT_SOCK="${SSH_AGENT_DIR}/agent@$(hostname |sed 's/\..*//')" + # launch a new agent + if [ -z "${SSH_AUTH_SOCK}" ]; then + [ ! -d "${SSH_AGENT_DIR}" ] \ + && mkdir -p "${SSH_AGENT_DIR}" 2>/dev/null\ + && chmod 0700 "${SSH_AGENT_DIR}" + # search for an already running agent + if ps wwx -o args |awk '$1 ~ "ssh-agent$" && $3 == "'"${SSH_AGENT_SOCK}"'"' |wc -l |grep -q 0; then + rm -f "${SSH_AGENT_SOCK}" + ssh-agent -a "${SSH_AGENT_SOCK}" >/dev/null 2>&1 fi - # attach to agent - export SSH_AUTH_SOCK="${SSH_AUTH_SOCK:-${SSH_AGENT_SOCK}}" - # list private keys to add - # shellcheck disable=SC2068 - for dir in ${@:-${HOME}/.ssh}; do - if [ "${SSH_ADD_RECURSIVE:-}" = true ]; then - GREP_RECURSIVE_FLAG="r" - else - GREP_RECURSIVE_CHAR="*" - fi - SSH_PRIVATE_KEYS="${SSH_PRIVATE_KEYS:-} ${dir}/id_rsa $(grep -l${GREP_RECURSIVE_FLAG:-} 'PRIVATE KEY' "${dir}"/"${GREP_RECURSIVE_CHAR:-}" 2>/dev/null |grep -vw "${dir}"/id_rsa)" - done - # shellcheck disable=SC2086 - printf '%s\n' ${SSH_PRIVATE_KEYS} |while read -r file; do - [ -r "${file}" ] || continue - # add private key to agent - ssh-add -l |grep -q "$(ssh-keygen -lf "${file}" 2>/dev/null |awk '{print $2}')" 2>/dev/null || ssh-add "${file}" - done - unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_AGENT_DIR SSH_AGENT_SOCK SSH_PRIVATE_KEYS + fi + # attach to agent + export SSH_AUTH_SOCK="${SSH_AUTH_SOCK:-${SSH_AGENT_SOCK}}" + # list private keys to add + # shellcheck disable=SC2068 + for dir in ${@:-${HOME}/.ssh}; do + if [ "${SSH_ADD_RECURSIVE:-}" = true ]; then + GREP_RECURSIVE_FLAG="r" + else + GREP_RECURSIVE_CHAR="*" + fi + SSH_PRIVATE_KEYS="${SSH_PRIVATE_KEYS:-} ${dir}/id_rsa $(grep -l${GREP_RECURSIVE_FLAG:-} 'PRIVATE KEY' "${dir}"/"${GREP_RECURSIVE_CHAR:-}" 2>/dev/null |grep -vw "${dir}"/id_rsa)" + done + # shellcheck disable=SC2086 + printf '%s\n' ${SSH_PRIVATE_KEYS} |while read -r file; do + [ -r "${file}" ] || continue + # add private key to agent + ssh-add -l |grep -q "$(ssh-keygen -lf "${file}" 2>/dev/null |awk '{print $2}')" 2>/dev/null || ssh-add "${file}" + done + unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_AGENT_DIR SSH_AGENT_SOCK SSH_PRIVATE_KEYS } ## ssh_del() removes all private keys in ~/.ssh/ from ssh agent ssh_del() { - command -v ssh-add >/dev/null 2>&1 || return - # attach to agent - if [ -z "${SSH_AUTH_SOCK}" ]; then - return + command -v ssh-add >/dev/null 2>&1 || return + # attach to agent + if [ -z "${SSH_AUTH_SOCK}" ]; then + return + fi + # list private keys to del + # shellcheck disable=SC2068 + for dir in ${@:-${HOME}/.ssh}; do + if [ "${SSH_DEL_RECURSIVE:-}" = true ]; then + GREP_RECURSIVE_FLAG="r" + else + GREP_RECURSIVE_CHAR="*" fi - # list private keys to del - # shellcheck disable=SC2068 - for dir in ${@:-${HOME}/.ssh}; do - if [ "${SSH_DEL_RECURSIVE:-}" = true ]; then - GREP_RECURSIVE_FLAG="r" - else - GREP_RECURSIVE_CHAR="*" - fi - SSH_PRIVATE_KEYS="${SSH_PRIVATE_KEYS:-} ${dir}/id_rsa $(grep -l${GREP_RECURSIVE_FLAG:-} 'PRIVATE KEY' "${dir}"/"${GREP_RECURSIVE_CHAR:-}" 2>/dev/null |grep -vw "${dir}"/id_rsa)" - done - # shellcheck disable=SC2086 - printf '%s\n' ${SSH_PRIVATE_KEYS} |while read -r file; do - [ -r "${file}" ] || continue - # remove private key from agent - ssh-add -l |grep -q "$(ssh-keygen -lf "${file}" 2>/dev/null |awk '{print $2}')" 2>/dev/null && ssh-add -d "${file}" - done - unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_PRIVATE_KEYS + SSH_PRIVATE_KEYS="${SSH_PRIVATE_KEYS:-} ${dir}/id_rsa $(grep -l${GREP_RECURSIVE_FLAG:-} 'PRIVATE KEY' "${dir}"/"${GREP_RECURSIVE_CHAR:-}" 2>/dev/null |grep -vw "${dir}"/id_rsa)" + done + # shellcheck disable=SC2086 + printf '%s\n' ${SSH_PRIVATE_KEYS} |while read -r file; do + [ -r "${file}" ] || continue + # remove private key from agent + ssh-add -l |grep -q "$(ssh-keygen -lf "${file}" 2>/dev/null |awk '{print $2}')" 2>/dev/null && ssh-add -d "${file}" + done + unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_PRIVATE_KEYS } ## tmux_attach() attaches existing tmux session or creates a new one tmux_attach() { - command -v tmux >/dev/null 2>&1 || return - TMUX_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" - if [ -z "${TMUX}" ]; then - printf 'Attaching tmux.' && sleep 1 && printf '.' && sleep 1 && printf '.' && sleep 1 - exec tmux -L"${TMUX_SESSION}" new-session -A -s"${TMUX_SESSION}" - fi - unset TMUX_SESSION + command -v tmux >/dev/null 2>&1 || return + TMUX_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" + if [ -z "${TMUX}" ]; then + printf 'Attaching tmux.' && sleep 1\ + && printf '.' && sleep 1\ + && printf '.' && sleep 1 + exec tmux -L"${TMUX_SESSION}" new-session -A -s"${TMUX_SESSION}" + fi + unset TMUX_SESSION } ## tmux_detach() detaches current tmux session tmux_detach() { - tmux detach + tmux detach } ## user_count() prints number of "users sessions"/"users"/"logged users" user_count() { - ps ax -o user,tty 2>/dev/null |awk ' - $2 ~ /^(pts|tty)/ {users_session++; logged[$1]++;}; - {count[$1]++;} - END { - for (uc in count) {c = c" "uc;}; users_count=split(c,v," ")-1; - for (ul in logged) {l = l" "ul;}; users_logged=split(l,v," ")-1; - print users_session+0"/"users_count"/"users_logged; - }' + ps ax -o user,tty 2>/dev/null |awk ' + $2 ~ /^(pts|tty)/ { users_session++; logged[$1]++; }; + { count[$1]++; } + END { + for (uc in count) { c = c" "uc; }; users_count=split(c,v," ")-1; + for (ul in logged) { l = l" "ul; }; users_logged=split(l,v," ")-1; + print users_session+0"/"users_count"/"users_logged; + }' } + +# vim:ts=2:sw=2:sts=2:et diff --git a/ansible/roles/hosts/templates/Makefile.j2 b/ansible/roles/hosts/templates/Makefile.j2 index 960472f..2ceb8ad 100644 --- a/ansible/roles/hosts/templates/Makefile.j2 +++ b/ansible/roles/hosts/templates/Makefile.j2 @@ -1,7 +1,5 @@ -USER := aynicos -APP ?= $(shell hostname -s) CMDS := exec -SERVICE ?= php +SERVICE ?= cli -include .env ansible: @@ -14,16 +12,16 @@ ansible-pull: @$(foreach var,$(ENV_USER_VARS),$(if $($(var)),$(var)='$($(var))')) $(if $(ANSIBLE_CONFIG),ANSIBLE_CONFIG=$(ANSIBLE_GIT_DIRECTORY)/$(ANSIBLE_CONFIG)) ansible-pull --url $(ANSIBLE_GIT_REPOSITORY) $(if $(ANSIBLE_GIT_KEY_FILE),--key-file $(ANSIBLE_GIT_KEY_FILE)) $(if $(ANSIBLE_GIT_VERSION),--checkout $(ANSIBLE_GIT_VERSION)) $(if $(ANSIBLE_GIT_DIRECTORY),--directory $(ANSIBLE_GIT_DIRECTORY)) $(if $(ANSIBLE_TAGS),--tags $(ANSIBLE_TAGS)) $(if $(ANSIBLE_EXTRA_VARS),--extra-vars '$(ANSIBLE_EXTRA_VARS)') $(if $(findstring true,$(FORCE)),--force) $(if $(findstring true,$(DRYRUN)),--check) --full $(if $(ANSIBLE_INVENTORY),--inventory $(ANSIBLE_INVENTORY)) $(ANSIBLE_VERBOSE) $(ANSIBLE_PLAYBOOK) exec: - @make -C $(ANSIBLE_GIT_DIRECTORY) exec ARGS='$(ARGS)' DOCKER_NAME=$(USER)_$(ENV)_$(APP)_$(SERVICE) ENV=$(ENV) DOCKER_RUN_WORKDIR= + @make -C $(ANSIBLE_GIT_DIRECTORY) exec ARGS='$(ARGS)' DOCKER_NAME=$(COMPOSE_PROJECT_NAME)_$(SERVICE) ENV=$(ENV) DOCKER_RUN_WORKDIR= connect: - @docker exec -it $(USER)_$(ENV)_$(APP)_$(SERVICE) /bin/bash || true + @docker exec -it $(COMPOSE_PROJECT_NAME)_$(SERVICE) /bin/bash || true logs: - @docker logs --follow --tail 100 $(USER)_$(ENV)_$(APP)_$(SERVICE) || true + @docker logs --follow --tail 100 $(COMPOSE_PROJECT_NAME)_$(SERVICE) || true logs-nofollow: - @docker logs --tail 100 $(USER)_$(ENV)_$(APP)_$(SERVICE) || true + @docker logs --tail 100 $(COMPOSE_PROJECT_NAME)_$(SERVICE) || true run: @$(ARGS) diff --git a/aws/aws.mk b/aws/aws.mk index 82c1096..29fdf64 100644 --- a/aws/aws.mk +++ b/aws/aws.mk @@ -1,6 +1,6 @@ # target aws: Fire docker-build-aws, Call aws ARGS .PHONY: aws -aws: docker-build-aws +aws: $(if $(DOCKER_RUN),docker-build-aws) $(call aws,$(ARGS)) # target aws-deploy: Call aws deploy create-deployment with application-name CODEDEPLOY_APP_NAME @@ -29,7 +29,7 @@ aws-ecr-get-login: .PHONY: aws-iam-create-role-% aws-iam-create-role-%: base docker-build-aws $(eval IGNORE_DRYRUN := true) - $(eval json := $(shell $(call exec,envsubst < aws/policies/$*-trust.json))) + $(eval json := $(shell $(call exec,sh -c 'envsubst < aws/policies/$*-trust.json'))) $(eval IGNORE_DRYRUN := false) $(call aws,iam create-role --role-name $* --assume-role-policy-document '$(json)') @@ -37,7 +37,7 @@ aws-iam-create-role-%: base docker-build-aws .PHONY: aws-iam-put-role-policy-% aws-iam-put-role-policy-%: base docker-build-aws $(eval IGNORE_DRYRUN := true) - $(eval json := $(shell $(call exec,envsubst < aws/policies/$*.json))) + $(eval json := $(shell $(call exec,sh -c 'envsubst < aws/policies/$*.json'))) $(eval IGNORE_DRYRUN := false) $(call aws,iam put-role-policy --role-name $* --policy-name $* --policy-document '$(json)') @@ -83,7 +83,7 @@ aws-s3api-get-head-object-lastmodified: docker-build-aws .PHONY: aws-ec2-import-snapshot aws-ec2-import-snapshot: base docker-build-aws aws-s3api-get-head-object-etag aws-s3api-get-head-object-lastmodified $(eval IGNORE_DRYRUN := true) - $(eval json := $(shell $(call exec,envsubst < aws/import-snapshot.json))) + $(eval json := $(shell $(call exec,sh -c 'envsubst < aws/import-snapshot.json'))) $(eval IGNORE_DRYRUN := false) $(eval AWS_TASK_ID := $(shell $(call aws,ec2 import-snapshot --description '$(AWS_SNAP_DESCRIPTION)' --output text --query ImportTaskId --disk-container '$(json)'))) echo ImportTaskId: $(AWS_TASK_ID) @@ -191,7 +191,7 @@ aws-ec2-wait-snapshot-completed-%: docker-build-aws .PHONY: aws-ec2-register-image aws-ec2-register-image: base docker-build-aws aws-ec2-get-import-snapshot-tasks-id $(eval IGNORE_DRYRUN := true) - $(eval json := $(shell $(call exec,envsubst < aws/register-image-device-mappings.json))) + $(eval json := $(shell $(call exec,sh -c 'envsubst < aws/register-image-device-mappings.json'))) $(eval IGNORE_DRYRUN := false) $(eval AWS_AMI_ID := $(shell $(call aws,ec2 register-image --name '$(AWS_AMI_NAME)' --description '$(AWS_AMI_DESCRIPTION)' --architecture x86_64 --root-device-name /dev/sda1 --virtualization-type hvm --block-device-mappings '$(json)') 2>/dev/null)) echo ImageId: $(AWS_AMI_ID) diff --git a/aws/def.aws.mk b/aws/def.aws.mk index ed272f5..b12ac7a 100644 --- a/aws/def.aws.mk +++ b/aws/def.aws.mk @@ -1,25 +1,21 @@ -AWS_ACCESS_KEY_ID := $(shell $(call conf,$(HOME)/.aws/credentials,$(or $(AWS_PROFILE),default),aws_access_key_id)) -AWS_AMI_DESCRIPTION ?= app: $(APP) branch: $(BRANCH) env: $(ENV) iso: $(AWS_S3_KEY) user: $(USER) version: $(VERSION) -AWS_AMI_NAME ?= $(USER)/$(ENV)/$(APP)/ami/$(VERSION)/$(shell date +%Y%m%dT%H%M%S) +AWS_ACCESS_KEY_ID := $(if $(AWS_CREDENTIALS),$(shell $(call conf,$(AWS_CREDENTIALS),$(or $(AWS_PROFILE),default),aws_access_key_id))) +AWS_AMI_DESCRIPTION ?= $(SERVICE_VERSION) +AWS_AMI_NAME ?= $(SERVICE_NAME)-$(AWS_S3_FILENAME) +AWS_CREDENTIALS ?= $(wildcard $(HOME)/.aws/credentials) AWS_DEFAULT_REGION ?= eu-west-1 AWS_DEFAULT_OUTPUT ?= text AWS_INSTANCE_ID ?= $(shell timeout 0.1 curl -s http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null) AWS_VM_IMPORT_ROLE_NAME ?= vmimport -AWS_S3_BUCKET ?= $(USER)-$(ENV)-config +AWS_S3_BUCKET ?= $(SERVICE_NAME) +AWS_S3_FILENAME ?= $(PACKER_ISO_NAME) AWS_S3_KEY ?= $(PACKER_ISO_FILE) -AWS_SECRET_ACCESS_KEY := $(shell $(call conf,$(HOME)/.aws/credentials,$(or $(AWS_PROFILE),default),aws_secret_access_key)) -AWS_SNAP_DESCRIPTION ?= iso: $(AWS_S3_KEY) env: $(ENV) app: $(APP) branch: $(BRANCH) version: $(VERSION) user: $(USER) etag: $(AWS_S3_KEY_ETAG) date: $(AWS_S3_KEY_DATE) +AWS_SECRET_ACCESS_KEY := $(if $(AWS_CREDENTIALS),$(shell $(call conf,$(AWS_CREDENTIALS),$(or $(AWS_PROFILE),default),aws_secret_access_key))) +AWS_SNAP_DESCRIPTION ?= $(SERVICE_NAME)-$(SERVICE_VERSION)-$(AWS_S3_FILENAME) CMDS += aws DOCKER_RUN_VOLUME += -v $(HOME)/.aws:/home/$(USER)/.aws ENV_VARS += AWS_ACCESS_KEY_ID AWS_AMI_DESCRIPTION AWS_AMI_NAME AWS_DEFAULT_OUTPUT AWS_DEFAULT_REGION AWS_INSTANCE_ID AWS_PROFILE AWS_S3_BUCKET AWS_S3_KEY AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_SNAP_DESCRIPTION AWS_SNAP_ID -ifeq ($(DOCKER), true) -define aws - $(call run,$(DOCKER_REPOSITORY)/aws:$(DOCKER_IMAGE_TAG) $(1)) -endef -else # function aws: Call run aws with arg 1 define aws - $(call run,aws $(1)) + $(RUN) $(call run,aws $(1),$(DOCKER_REPOSITORY)/) endef -endif diff --git a/docker/ssh/Dockerfile b/docker/ssh/Dockerfile index d09e576..41df29f 100644 --- a/docker/ssh/Dockerfile +++ b/docker/ssh/Dockerfile @@ -13,8 +13,6 @@ ENV SOCKET_DIR /tmp/ssh-agent ENV SSH_AUTH_SOCK ${SOCKET_DIR}/socket ENV SSH_AUTH_PROXY_SOCK ${SOCKET_DIR}/proxy-socket -VOLUME ${SOCKET_DIR} - # Copy entrypoint script to container COPY ${DOCKER_BUILD_DIR}/docker-entrypoint.sh /docker-entrypoint.sh @@ -51,4 +49,7 @@ RUN let $UID >/dev/null 2>&1 \ && chown $UID:$GID /home/$USER \ || true +RUN mkdir -p $SOCKET_DIR && chown $USER $SOCKET_DIR + +VOLUME ${SOCKET_DIR} USER $USER diff --git a/docker/theia/rc_functions.sh b/docker/theia/rc_functions.sh index 70951cb..8ab8678 100644 --- a/docker/theia/rc_functions.sh +++ b/docker/theia/rc_functions.sh @@ -1,124 +1,283 @@ -# force a command to run and restart it when it exits -force () { - PS_X_FIELD=1 - if [ $# -gt 0 ]; then - # awk expression to match $@ - while true; do - [ $(ps wwx -o args |awk ' - BEGIN {nargs=split("'"$*"'",args)} - $field == args[1] { - matched=1; - for (i=1;i<=NF-field;i++) { - if ($(i+field) == args[i+1]) {matched++} - } - if (matched == nargs) {found++} - } - END {print found+0}' field="${PS_X_FIELD}") -eq 0 ] \ - && "$@" || sleep 1; - done - fi -} +# shellcheck shell=sh +## rc_function.sh defines customs shell functions +# author: Yann "aya" Autissier +# license: MIT +# updated: 2021/03/04 -# start an ssh agent and add any private key in ~/.ssh -ssh_agent () { - command -v ssh-agent >/dev/null 2>&1 && command -v ssh-add >/dev/null 2>&1 || return - SSH_AGENT_DIR="/tmp/ssh-$(id -u)" - SSH_AGENT_SOCK="${SSH_AGENT_DIR}/agent@$(hostname |sed 's/\..*//')" - [ -z "${SSH_AUTH_SOCK}" ] \ - && { [ -d "${SSH_AGENT_DIR}" ] || { mkdir "${SSH_AGENT_DIR}" 2>/dev/null && chmod 0700 "${SSH_AGENT_DIR}"; } } \ - && [ $(ps wwx -o args |awk '$1 == "ssh-agent" && $3 == "'"${SSH_AGENT_SOCK}"'"' |wc -l) -eq 0 ] \ - && rm -f "${SSH_AGENT_SOCK}" \ - && ssh-agent -a "${SSH_AGENT_SOCK}" >/dev/null 2>&1 - export SSH_AUTH_SOCK="${SSH_AUTH_SOCK:-${SSH_AGENT_SOCK}}" - (echo "${HOME}"/.ssh/id_rsa; grep -l 'PRIVATE KEY' "${HOME}"/.ssh/* |grep -vE "^${HOME}/.ssh/id_rsa$") |while read -r file; do - [ -r "${file}" ] && [ -z "$(ssh-add -l |awk '{print $3}' |grep -E "^${file}$")" ] && ssh-add "${file}" +## force() runs a command sine die +force() { + if [ $# -gt 0 ]; then + while true; do + "$@" + sleep 1 done - unset SSH_AGENT_DIR SSH_AGENT_SOCK + fi } -# attach an existing screen or create a new one -attach_screen () { - command -v screen >/dev/null 2>&1 || return - if [ -z "${STY}" ]; then - # attach screen in tmux window 0 - [ -n "${TMUX}" ] && [ "$(tmux list-window 2>/dev/null |awk '$NF == "(active)" {print $1}' |sed 's/:$//')" != "0" ] && return - /bin/echo -n 'Attaching screen.' && sleep 1 && /bin/echo -n '.' && sleep 1 && /bin/echo -n '.' && sleep 1 && screen -xRR -S "$(id -nu)" 2>/dev/null - fi +## force8() runs a command sine die if not already running +force8() { + if [ $# -gt 0 ]; then + while true; do + # awk expression to match $@ + [ "$(ps wwx -o args 2>/dev/null |awk -v field="${PS_X_FIELD:-1}" ' + BEGIN { nargs=split("'"$*"'",args); } + # first field matched + $field == args[1] { + matched=1; + # match following fields + for (i=1;i<=NF-field;i++) { + if ($(i+field) == args[i+1]) { matched++; } + }; + # all fields matched + if (matched == nargs) { found++; } + } + END { print found+0; }' + )" = 0 ] && "$@" + sleep 1 + done + fi } -# attach an existing tmux or create a new one -attach_tmux () { - command -v tmux >/dev/null 2>&1 || return - SESSION_NAME="$(id -nu)@$(hostname |sed 's/\..*//')" - if [ -z "${TMUX}" ]; then - /bin/echo -n 'Attaching tmux.' && sleep 1 && /bin/echo -n '.' && sleep 1 && /bin/echo -n '.' && sleep 1 && tmux -L"${SESSION_NAME}" -q has-session >/dev/null 2>&1 && tmux -L"${SESSION_NAME}" attach-session -d || tmux -L"${SESSION_NAME}" new-session -s"${SESSION_NAME}" - fi +## load_average() prints the current load average +load_average() { + awk '{printf "%.1f\n" $1}' /proc/loadavg 2>/dev/null\ + || uptime 2>/dev/null |awk '{printf "%.1f\n", $(NF-2)}' + } + +## process_count() prints number of "processes"/"running processes"/"D-state" +process_count() { + ps ax -o stat 2>/dev/null |awk ' + $1 ~ /R/ {process_running++}; + $1 ~ /D/ {process_dstate++}; + END { print NR-1"/"process_running+0"/"process_dstate+0; }' } -# echo the "number of running processes"/"total number of processes"/"number of processes in D-state" -process_count () { - ps ax -o stat 2>/dev/null |awk '$1 ~ /R/ {process_running++}; $1 ~ /D/ {process_dstate++}; END {print process_running+0"/"NR-1"/"process_dstate+0}' +## prompt_set() exports custom PROMPT_COMMAND +prompt_set() { + case "${TERM}" in + screen*) + ESCAPE_CODE_DCS="\033k" + ESCAPE_CODE_ST="\033\\" + ;; + linux*|xterm*|rxvt*) + ESCAPE_CODE_DCS="\033]0;" + ESCAPE_CODE_ST="\007" + ;; + *) + ;; + esac + # in a screen + if [ -n "${STY}" ]; then + export PROMPT_COMMAND='printf\ + "${ESCAPE_CODE_DCS:-\033]0;}%s${ESCAPE_CODE_ST:-\007}"\ + "${PWD##*/}"' + else + export PROMPT_COMMAND='printf\ + "${ESCAPE_CODE_DCS:-\033]0;}%s@%s:%s${ESCAPE_CODE_ST:-\007}"\ + "${USER}"\ + "${HOSTNAME%%.*}"\ + "${PWD##*/}"' + fi + unset ESCAPE_CODE_DCS ESCAPE_CODE_ST } -# echo the "number of distinct logged in users"/"number of distinct users running processes"/"number of logged in users" -user_count () { - ps ax -o user,tty,comm 2>/dev/null |awk '$2 !~ /^\?/ && $3 !~ /getty$/ {logged[$1]++; tty[$2]++}; {user[$1]++}; END {for (ul in logged) {sl = sl" "ul;}; users_logged=split(sl,a," ")-1; for (uu in user) {su = su" "uu;}; users_user=split(su,a," ")-1; for (ut in tty) {st = st" "ut;}; users_tty=split(st,a," ")-1; print users_logged"/"users_user"/"users_tty}' -} +## ps1_set() exports custom PS1 +ps1_set() { + case "$0" in + *sh) + COLOR_DGRAY="\[\033[1;30m\]" + COLOR_RED="\[\033[01;31m\]" + COLOR_GREEN="\[\033[01;32m\]" + COLOR_BROWN="\[\033[0;33m\]" + COLOR_YELLOW="\[\033[01;33m\]" + COLOR_BLUE="\[\033[01;34m\]" + COLOR_CYAN="\[\033[0;36m\]" + COLOR_GRAY="\[\033[0;37m\]" + COLOR_RESET="\[\033[0m\]" + ;; + *) + ;; + esac -# echo the load average -load_average () { - awk '{print $1}' /proc/loadavg 2>/dev/null || uptime 2>/dev/null |awk '{print $(NF-2)}' -} - -# export PS1 -custom_ps1 () { - case "$0" in - *ash) - local DGRAY="\[\033[1;30m\]" - local RED="\[\033[01;31m\]" - local GREEN="\[\033[01;32m\]" - local BROWN="\[\033[0;33m\]" - local YELLOW="\[\033[01;33m\]" - local BLUE="\[\033[01;34m\]" - local CYAN="\[\033[0;36m\]" - local GRAY="\[\033[0;37m\]" - local NC="\[\033[0m\]" - ;; + PS1_STATUS="\$?" + PS1_COUNT="${COLOR_DGRAY}[\` + case \"$PS1_STATUS\" in + 0) + printf \"${COLOR_BLUE}${PS1_STATUS}\";; + 1) + printf \"${COLOR_YELLOW}${PS1_STATUS}\";; *) - ;; + printf \"${COLOR_RED}${PS1_STATUS}\";; esac - - local COLOR="\$([ \"\$(id -u)\" = 0 ] && echo \"${RED}\" || echo \"${BROWN}\")" - local COUNT="${DGRAY}[${BLUE}\$(user_count 2>/dev/null)${DGRAY}|${BLUE}\$(process_count 2>/dev/null)${DGRAY}|${BLUE}\$(load_average 2>/dev/null)${DGRAY}]" - local END="\$([ \"\$(id -u)\" = 0 ] && echo \"#\" || echo \"\$\")" - local HOSTNAME="\$(hostname |sed 's/\..*//')" - - type __git_ps1 >/dev/null 2>&1 \ - && local GIT="\$(__git_ps1 2>/dev/null \" (%s)\")" \ - || local GIT="\$(BRANCH=\$(git rev-parse --abbrev-ref HEAD 2>/dev/null); [ -n \"\${BRANCH}\" ] && echo \" (\${BRANCH})\")" - - local USER="\$(id -nu)" - local WORKDIR="\$(pwd |sed 's|^'\${HOME}'\(/.*\)*$|~\1|')" - - export PS1="${COUNT}${COLOR}${USER}${DGRAY}@${CYAN}${HOSTNAME}${DGRAY}:${GRAY}${WORKDIR}${CYAN}${GIT}${DGRAY}${END}${NC} " + type process_count >/dev/null 2>&1 && printf\ + \"${COLOR_DGRAY}|${COLOR_BLUE}%s\"\ + \"\$(process_count 2>/dev/null)\" + type user_count >/dev/null 2>&1 && printf\ + \"${PS1_COUNT}${COLOR_DGRAY}|${COLOR_BLUE}%s\"\ + \"\$(user_count 2>/dev/null)\" + type load_average >/dev/null 2>&1 && printf\ + \"${PS1_COUNT}${COLOR_DGRAY}|${COLOR_BLUE}%s\"\ + \"\$(load_average 2>/dev/null)\" + \`${COLOR_DGRAY}]${COLOR_RESET}" + PS1_END="${COLOR_DGRAY}\$( + if [ \"\$(id -u)\" = 0 ]; then + printf \"#\"; + else + printf \"\$\"; + fi + )${COLOR_RESET}" + PS1_GIT="\$( + if type __git_ps1 >/dev/null 2>&1; then + printf \"\$(__git_ps1 2>/dev/null \" (%s)\")\" + else + printf \"\$(BRANCH=\$(git rev-parse --abbrev-ref HEAD 2>/dev/null);\ + [ -n \"\${BRANCH}\" ] && printf \" (\${BRANCH})\")\" + fi + )" + PS1_GIT="${COLOR_CYAN}${PS1_GIT}${COLOR_RESET}" + PS1_HOSTNAME_COLOR="\`case \"\${ENV}\" in + [Pp][Rr][0Oo][Dd]*) + printf \"${COLOR_RED}\";; + *) + if [ -n \"\${ENV}\" ]; then + printf \"${COLOR_YELLOW}\"; + else + printf \"${COLOR_GREEN}\"; + fi;; + esac\`" + PS1_HOSTNAME="${PS1_HOSTNAME_COLOR}\$(hostname |sed 's/\..*//')${COLOR_RESET}" + PS1_USER_COLOR="\$( + if [ \"\$(id -u)\" = 0 ]; then + printf \"${COLOR_RED}\"; + else + printf \"${COLOR_BROWN}\"; + fi + )" + PS1_USER="${PS1_USER_COLOR}\$(id -nu):\$(id -u)${COLOR_RESET}" + PS1_WORKDIR="${COLOR_GRAY}\$( + pwd |sed 's|^'\${HOME}'\(/.*\)*$|~\1|' + )${COLOR_RESET}" + PS1="${PS1_COUNT}${PS1_USER}${COLOR_DGRAY}@${PS1_HOSTNAME}" + PS1="${PS1}${COLOR_DGRAY}:${PS1_WORKDIR}${PS1_GIT}${PS1_END} " + export 'PS1' + unset PS1_COUNT PS1_END PS1_GIT PS1_HOSTNAME PS1_HOSTNAME_COLOR\ + PS1_USER PS1_USER_COLOR PS1_STATUS PS1_WORKDIR } -# export PROMPT_COMMAND -custom_prompt () { - case "${TERM}" in - screen*) - ESCAPE_CODE_DCS="\033k" - ESCAPE_CODE_ST="\033\\" - ;; - linux*|xterm*|rxvt*) - ESCAPE_CODE_DCS="\033]0;" - ESCAPE_CODE_ST="\007" - ;; - *) - ;; - esac - # in a screen - [ -n "${STY}" ] \ - && export PROMPT_COMMAND='printf "${ESCAPE_CODE_DCS:-\033]0;}%s${ESCAPE_CODE_ST:-\007}" "${PWD##*/}"' \ - || export PROMPT_COMMAND='printf "${ESCAPE_CODE_DCS:-\033]0;}%s@%s:%s${ESCAPE_CODE_ST:-\007}" "${USER}" "${HOSTNAME%%.*}" "${PWD##*/}"' +## screen_attach() attaches existing screen session or creates a new one +screen_attach() { + command -v screen >/dev/null 2>&1 || return + SCREEN_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" + if [ -z "${STY}" ]; then + # attach screen in tmux window 0 only ;) + [ -n "${TMUX}" ] \ + && [ "$(tmux list-window 2>/dev/null |awk '$NF == "(active)" {print $1}'\ + |sed 's/:$//')" != "0" ] \ + && return + printf 'Attaching screen.' && sleep 1\ + && printf '.' && sleep 1\ + && printf '.' && sleep 1 + exec screen -xRR -S "${SCREEN_SESSION}" + fi + unset SCREEN_SESSION } + +## screen_detach() detaches current screen session +screen_detach() { + screen -d +} + +## ssh_add() loads all private keys in ~/.ssh/ to ssh agent +ssh_add() { + command -v ssh-agent >/dev/null 2>&1 && command -v ssh-add >/dev/null 2>&1 || return + SSH_AGENT_DIR="/tmp/ssh-$(id -u)" + SSH_AGENT_SOCK="${SSH_AGENT_DIR}/agent@$(hostname |sed 's/\..*//')" + # launch a new agent + if [ -z "${SSH_AUTH_SOCK}" ]; then + [ ! -d "${SSH_AGENT_DIR}" ] \ + && mkdir -p "${SSH_AGENT_DIR}" 2>/dev/null\ + && chmod 0700 "${SSH_AGENT_DIR}" + # search for an already running agent + if ps wwx -o args |awk '$1 ~ "ssh-agent$" && $3 == "'"${SSH_AGENT_SOCK}"'"' |wc -l |grep -q 0; then + rm -f "${SSH_AGENT_SOCK}" + ssh-agent -a "${SSH_AGENT_SOCK}" >/dev/null 2>&1 + fi + fi + # attach to agent + export SSH_AUTH_SOCK="${SSH_AUTH_SOCK:-${SSH_AGENT_SOCK}}" + # list private keys to add + # shellcheck disable=SC2068 + for dir in ${@:-${HOME}/.ssh}; do + if [ "${SSH_ADD_RECURSIVE:-}" = true ]; then + GREP_RECURSIVE_FLAG="r" + else + GREP_RECURSIVE_CHAR="*" + fi + SSH_PRIVATE_KEYS="${SSH_PRIVATE_KEYS:-} ${dir}/id_rsa $(grep -l${GREP_RECURSIVE_FLAG:-} 'PRIVATE KEY' "${dir}"/"${GREP_RECURSIVE_CHAR:-}" 2>/dev/null |grep -vw "${dir}"/id_rsa)" + done + # shellcheck disable=SC2086 + printf '%s\n' ${SSH_PRIVATE_KEYS} |while read -r file; do + [ -r "${file}" ] || continue + # add private key to agent + ssh-add -l |grep -q "$(ssh-keygen -lf "${file}" 2>/dev/null |awk '{print $2}')" 2>/dev/null || ssh-add "${file}" + done + unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_AGENT_DIR SSH_AGENT_SOCK SSH_PRIVATE_KEYS +} + +## ssh_del() removes all private keys in ~/.ssh/ from ssh agent +ssh_del() { + command -v ssh-add >/dev/null 2>&1 || return + # attach to agent + if [ -z "${SSH_AUTH_SOCK}" ]; then + return + fi + # list private keys to del + # shellcheck disable=SC2068 + for dir in ${@:-${HOME}/.ssh}; do + if [ "${SSH_DEL_RECURSIVE:-}" = true ]; then + GREP_RECURSIVE_FLAG="r" + else + GREP_RECURSIVE_CHAR="*" + fi + SSH_PRIVATE_KEYS="${SSH_PRIVATE_KEYS:-} ${dir}/id_rsa $(grep -l${GREP_RECURSIVE_FLAG:-} 'PRIVATE KEY' "${dir}"/"${GREP_RECURSIVE_CHAR:-}" 2>/dev/null |grep -vw "${dir}"/id_rsa)" + done + # shellcheck disable=SC2086 + printf '%s\n' ${SSH_PRIVATE_KEYS} |while read -r file; do + [ -r "${file}" ] || continue + # remove private key from agent + ssh-add -l |grep -q "$(ssh-keygen -lf "${file}" 2>/dev/null |awk '{print $2}')" 2>/dev/null && ssh-add -d "${file}" + done + unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_PRIVATE_KEYS +} + +## tmux_attach() attaches existing tmux session or creates a new one +tmux_attach() { + command -v tmux >/dev/null 2>&1 || return + TMUX_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" + if [ -z "${TMUX}" ]; then + printf 'Attaching tmux.' && sleep 1\ + && printf '.' && sleep 1\ + && printf '.' && sleep 1 + exec tmux -L"${TMUX_SESSION}" new-session -A -s"${TMUX_SESSION}" + fi + unset TMUX_SESSION +} + +## tmux_detach() detaches current tmux session +tmux_detach() { + tmux detach +} + +## user_count() prints number of "users sessions"/"users"/"logged users" +user_count() { + ps ax -o user,tty 2>/dev/null |awk ' + $2 ~ /^(pts|tty)/ { users_session++; logged[$1]++; }; + { count[$1]++; } + END { + for (uc in count) { c = c" "uc; }; users_count=split(c,v," ")-1; + for (ul in logged) { l = l" "ul; }; users_logged=split(l,v," ")-1; + print users_session+0"/"users_count"/"users_logged; + }' +} + +# vim:ts=2:sw=2:sts=2:et diff --git a/make/apps/build.mk b/make/apps/build.mk index bc6fefb..2383311 100644 --- a/make/apps/build.mk +++ b/make/apps/build.mk @@ -9,9 +9,9 @@ build-env: bootstrap $(call docker-compose-exec,$(SERVICE), \ rm -f .env \ && make .env ENV=$(ENV) \ - && echo BUILD=true >> .env \ + && printf 'BUILD=true\n' >> .env \ && $(foreach var,$(BUILD_ENV_VARS), \ - $(if $($(var)),sed -i '/^$(var)=/d' .env && echo $(var)='$($(var))' >> .env &&) \ + $(if $($(var)),sed -i '/^$(var)=/d' .env && printf "$(var)='$($(var))'\n" >> .env &&) \ ) true \ ) diff --git a/make/apps/common.mk b/make/apps/common.mk index 5114225..7cd68c7 100644 --- a/make/apps/common.mk +++ b/make/apps/common.mk @@ -37,11 +37,9 @@ build@%: myos-base $(eval build_app := $(or $(filter $(DOCKER_BUILD_CACHE),false),$(filter-out $(docker_images),$(SERVICES)))) $(if $(build_app), \ $(call make,build-init app-build), \ - $(if $(VERBOSE), \ - $(foreach service,$(SERVICES), \ - echo "docker image $(DOCKER_REPOSITORY)/$(service):$(DOCKER_IMAGE_TAG) has id $(shell docker images -q $(DOCKER_REPOSITORY)/$(service):$(DOCKER_IMAGE_TAG) 2>/dev/null)" && \ - ) true \ - ) \ + $(foreach service,$(SERVICES), \ + $(call INFO,docker image $(DOCKER_REPOSITORY)/$(service):$(DOCKER_IMAGE_TAG) has id $(shell docker images -q $(DOCKER_REPOSITORY)/$(service):$(DOCKER_IMAGE_TAG) 2>/dev/null)) && \ + ) true \ ) # target clean: Clean application and docker images @@ -85,7 +83,7 @@ down: docker-compose-down ## Remove application dockers .PHONY: exec exec: ## Exec command in docker SERVICE ifneq (,$(filter $(ENV),$(ENV_DEPLOY))) - $(call exec,$(ARGS)) + $(RUN) $(call exec,$(ARGS)) else $(call make,docker-compose-exec,,ARGS) endif @@ -226,4 +224,4 @@ upgrade: update app-upgrade release-upgrade ## Upgrade application # target %-rule-exists: Print a warning message if % target does not exists %-rule-exists: - $(if $(filter $*,$(MAKECMDGOALS)),$(if $(filter-out $*,$(MAKE_TARGETS)),printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}target${COLOR_RESET} $* ${COLOR_GREEN}not available in app${COLOR_RESET} $(APP).\n" >&2)) + $(if $(filter $*,$(MAKECMDGOALS)),$(if $(filter-out $*,$(MAKE_TARGETS)),$(call WARNING,no target,$*,$(APP)))) diff --git a/make/apps/def.build.mk b/make/apps/def.build.mk index 28e3b08..47b2b50 100644 --- a/make/apps/def.build.mk +++ b/make/apps/def.build.mk @@ -1,5 +1,5 @@ BUILD_AUTHOR ?= $(DOCKER_AUTHOR) -BUILD_DATE ?= $(shell TZ=UTC date "+%d/%m/%YT%H:%M:%SZ" 2>/dev/null) +BUILD_DATE ?= $(shell TZ=UTC date "+%Y%m%dT%H%M%SZ" 2>/dev/null) BUILD_DESCRIPTION ?= Lot of Love BUILD_DOCUMENTATION ?= $(if $(wildcard README.md),$(APP_REPOSITORY)/blob/$(COMMIT)/README.md) BUILD_ENV_VARS ?= APP BRANCH BUILD_DATE BUILD_STATUS COMMIT DEPLOY_HOOK_URL ENV VERSION @@ -42,6 +42,7 @@ org.opencontainers.image.authors ?= $(BUILD_AUTHOR) org.opencontainers.image.licenses ?= $(BUILD_LICENSE) os.my.author ?= $(BUILD_AUTHOR) os.my.date ?= $(BUILD_DATE) +os.my.license ?= $(BUILD_LICENSE) os.my.name ?= $(BUILD_NAME) os.my.status ?= $(BUILD_STATUS) os.my.user ?= $(USER) diff --git a/make/apps/def.docker.mk b/make/apps/def.docker.mk index c65893a..be9cdbc 100644 --- a/make/apps/def.docker.mk +++ b/make/apps/def.docker.mk @@ -9,13 +9,8 @@ COMPOSE_FILE_SUBREPO ?= true else COMPOSE_FILE_APP ?= true endif -ifneq (,$(filter $(ENV),$(ENV_DEPLOY))) -COMPOSE_FILE_TMPFS ?= false -else -COMPOSE_FILE_TMPFS ?= true -endif COMPOSE_IGNORE_ORPHANS ?= false -COMPOSE_PROJECT_NAME ?= $(USER)_$(ENV)_$(APP) +COMPOSE_PROJECT_NAME ?= $(APP_ENV)$(subst /,,$(subst -,,$(APP_PATH))) COMPOSE_SERVICE_NAME ?= $(subst _,-,$(COMPOSE_PROJECT_NAME)) CONTEXT += COMPOSE_FILE DOCKER_REPOSITORY CONTEXT_DEBUG += DOCKER_BUILD_TARGET DOCKER_IMAGE_TAG DOCKER_REGISTRY DOCKER_SERVICE DOCKER_SERVICES @@ -29,7 +24,7 @@ DOCKER_BUILD_NO_CACHE ?= false DOCKER_BUILD_TARGET ?= $(if $(filter $(ENV),$(DOCKER_BUILD_TARGETS)),$(ENV),$(DOCKER_BUILD_TARGET_DEFAULT)) DOCKER_BUILD_TARGET_DEFAULT ?= master DOCKER_BUILD_TARGETS ?= $(ENV_DEPLOY) -DOCKER_BUILD_VARS ?= APP BRANCH DOCKER_GID DOCKER_REPOSITORY GID GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME SSH_REMOTE_HOSTS TARGET UID USER VERSION +DOCKER_BUILD_VARS ?= APP BRANCH DOCKER_GID DOCKER_REPOSITORY GID GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PRIVATE_IP_RANGE SSH_PUBLIC_HOST_KEYS SSH_REMOTE_HOSTS UID USER VERSION DOCKER_COMPOSE_DOWN_OPTIONS ?= DOCKER_COMPOSE_UP_OPTIONS ?= -d DOCKER_GID ?= $(call gid,docker) @@ -43,15 +38,28 @@ DOCKER_PLUGIN_S3FS_OPTIONS ?= allow_other,nonempty,use_path_request_style,u DOCKER_PLUGIN_S3FS_SECRETKEY ?= $(AWS_SECRET_ACCESS_KEY) DOCKER_PLUGIN_S3FS_REGION ?= eu-west-1 DOCKER_PLUGIN_VARS ?= S3FS_ACCESSKEY S3FS_OPTIONS S3FS_SECRETKEY S3FS_REGION -DOCKER_REGISTRY ?= registry +DOCKER_REGISTRY ?= docker.io DOCKER_REGISTRY_USERNAME ?= $(USER) DOCKER_REGISTRY_REPOSITORY ?= $(addsuffix /,$(DOCKER_REGISTRY))$(subst $(USER),$(DOCKER_REGISTRY_USERNAME),$(DOCKER_REPOSITORY)) DOCKER_REPOSITORY ?= $(subst _,/,$(COMPOSE_PROJECT_NAME)) DOCKER_SERVICE ?= $(lastword $(DOCKER_SERVICES)) -DOCKER_SERVICES ?= $(eval IGNORE_DRYRUN := true)$(eval IGNORE_VERBOSE := true)$(shell $(call docker-compose,--log-level critical config --services))$(eval IGNORE_DRYRUN := false)$(eval IGNORE_VERBOSE := false) +DOCKER_SERVICES ?= $(eval IGNORE_DRYRUN := true)$(shell $(call docker-compose,--log-level critical config --services))$(eval IGNORE_DRYRUN := false) DOCKER_SHELL ?= $(SHELL) ENV_VARS += COMPOSE_PROJECT_NAME COMPOSE_SERVICE_NAME DOCKER_BUILD_TARGET DOCKER_GID DOCKER_IMAGE_TAG DOCKER_REGISTRY DOCKER_REPOSITORY DOCKER_SHELL +ifeq ($(DOCKER), true) +DOCKER_COMPOSE ?= docker/compose:$(COMPOSE_VERSION) +else +DOCKER_COMPOSE ?= $(or $(shell docker compose >/dev/null 2>&1 && printf 'docker compose\n'),docker-compose) +endif + +ifeq ($(DRONE), true) +APP_PATH_PREFIX := $(DRONE_BUILD_NUMBER) +DOCKER_BUILD_CACHE := false +DOCKER_COMPOSE_DOWN_OPTIONS := --rmi all -v +DOCKER_COMPOSE_UP_OPTIONS := -d --build +endif + # https://github.com/docker/libnetwork/pull/2348 ifeq ($(HOST_SYSTEM), DARWIN) DOCKER_HOST_IFACE ?= $(shell docker run --rm -it --net=host alpine /sbin/ip -4 route list match 0/0 2>/dev/null |awk '{print $$5}' |awk '!seen[$$0]++' |head -1) @@ -65,43 +73,17 @@ DOCKER_INTERNAL_DOCKER_GATEWAY ?= $(shell /sbin/ip -4 route list match 0/0 2>/d DOCKER_INTERNAL_DOCKER_HOST ?= $(shell /sbin/ip addr show docker0 2>/dev/null |awk '$$1 == "inet" {sub(/\/.*/,"",$$2); print $$2}') endif -ifeq ($(DRONE), true) -APP_PATH_PREFIX := $(DRONE_BUILD_NUMBER) -COMPOSE_PROJECT_NAME := $(USER)_$(ENV)$(APP_PATH_PREFIX)_$(APP) -COMPOSE_SERVICE_NAME := $(subst _,-,$(COMPOSE_PROJECT_NAME)) -DOCKER_BUILD_CACHE := false -DOCKER_COMPOSE_DOWN_OPTIONS := --rmi all -v -DOCKER_COMPOSE_UP_OPTIONS := -d --build -DOCKER_REPOSITORY := $(USER)/$(ENV)/$(APP) -endif - -ifeq ($(DOCKER), true) - # function docker-compose: Run docker-compose with arg 1 define docker-compose $(call INFO,docker-compose,$(1)) - $(call run,docker/compose:$(COMPOSE_VERSION) $(patsubst %,-f %,$(COMPOSE_FILE)) -p $(COMPOSE_PROJECT_NAME) $(1)) + $(call run,$(DOCKER_COMPOSE) $(patsubst %,-f %,$(COMPOSE_FILE)) -p $(COMPOSE_PROJECT_NAME) $(1)) endef # function docker-compose-exec: Run docker-compose-exec with arg 2 in service 1 define docker-compose-exec $(call INFO,docker-compose-exec,$(1)$(comma) $(2)) - $(call run,docker/compose:$(COMPOSE_VERSION) $(patsubst %,-f %,$(COMPOSE_FILE)) -p $(COMPOSE_PROJECT_NAME) exec -T $(1) sh -c '$(2)') + $(call run,$(DOCKER_COMPOSE) $(patsubst %,-f %,$(COMPOSE_FILE)) -p $(COMPOSE_PROJECT_NAME) exec -T $(1) sh -c '$(2)') endef -else - -SHELL := /bin/bash -define docker-compose - $(call INFO,docker-compose,$(1)) - $(call run,docker-compose $(patsubst %,-f %,$(COMPOSE_FILE)) -p $(COMPOSE_PROJECT_NAME) $(1)) -endef -define docker-compose-exec - $(call INFO,docker-compose-exec,$(1)$(comma) $(2)) - $(call run,docker-compose $(patsubst %,-f %,$(COMPOSE_FILE)) -p $(COMPOSE_PROJECT_NAME) exec -T $(1) sh -c '$(2)') -endef - -endif - # function docker-build: Build docker image define docker-build $(call INFO,docker-build,$(1)$(comma) $(2)$(comma) $(3)) @@ -110,7 +92,7 @@ define docker-build $(eval target := $(subst ",,$(subst ',,$(or $(3),$(DOCKER_BUILD_TARGET))))) $(eval image_id := $(shell docker images -q $(tag) 2>/dev/null)) $(eval build_image := $(or $(filter false,$(DOCKER_BUILD_CACHE)),$(if $(image_id),,true))) - $(if $(build_image),$(RUN) docker build $(DOCKER_BUILD_ARGS) --build-arg DOCKER_BUILD_DIR="$(path)" $(DOCKER_BUILD_LABEL) --tag $(tag) $(if $(target),--target $(target)) -f $(path)/Dockerfile .,$(if $(VERBOSE),echo "docker image $(tag) has id $(image_id)",true)) + $(if $(build_image),$(RUN) docker build $(DOCKER_BUILD_ARGS) --build-arg DOCKER_BUILD_DIR="$(path)" $(DOCKER_BUILD_LABEL) --tag $(tag) $(if $(target),--target $(target)) -f $(path)/Dockerfile .,$(call INFO,docker image $(tag) has id $(image_id))) endef # function docker-commit: Commit docker image define docker-commit diff --git a/make/apps/def.mk b/make/apps/def.mk index 90c610b..0912f9b 100644 --- a/make/apps/def.mk +++ b/make/apps/def.mk @@ -1,5 +1,6 @@ APP_DIR ?= $(CURDIR) APP_DOMAIN ?= $(ENV)$(addprefix .,$(DOMAIN)) +APP_ENV ?= $(USER_ENV)_$(APP) APP_HOST ?= $(APP)$(addprefix .,$(APP_DOMAIN)) APP_INSTALLED ?= $(APPS) APP_PARENT ?= $(MONOREPO) @@ -11,10 +12,13 @@ APP_SCHEME ?= https APP_UPSTREAM_REPOSITORY ?= $(or $(shell git config --get remote.upstream.url 2>/dev/null),$(GIT_UPSTREAM_REPOSITORY)) APP_URI ?= $(APP_HOST)$(APP_PATH) APP_URL ?= $(APP_SCHEME)://$(APP_URI) -CONTEXT_DEBUG += APP_DIR APP_DOMAIN APP_HOST APP_PATH APP_URL APP_REPOSITORY APP_UPSTREAM_REPOSITORY ENV_DEPLOY +CONTEXT_DEBUG += APP_DIR APP_URL APP_REPOSITORY APP_UPSTREAM_REPOSITORY ENV_DEPLOY ENV_DEPLOY ?= $(shell ls .git/refs/remotes/origin/ 2>/dev/null) ENV_VARS += APP_DIR APP_DOMAIN APP_HOST APP_PATH APP_URL CONSUL_HTTP_TOKEN $(if $(filter true,$(MOUNT_NFS)),NFS_CONFIG) MOUNT_NFS ?= false NFS_CONFIG ?= addr=$(NFS_HOST),actimeo=3,intr,noacl,noatime,nocto,nodiratime,nolock,soft,rsize=32768,wsize=32768,tcp,rw,vers=3 NFS_HOST ?= host.docker.internal +SERVICE_ENV ?= $(subst _,-,$(APP_ENV)) +SERVICE_NAME ?= $(COMPOSE_SERVICE_NAME) +SERVICE_VERSION ?= $(BUILD_DATE)-$(VERSION) SERVICES ?= $(DOCKER_SERVICES) diff --git a/make/apps/docker.mk b/make/apps/docker.mk index 4ceb46e..ad0f55d 100644 --- a/make/apps/docker.mk +++ b/make/apps/docker.mk @@ -27,7 +27,7 @@ docker-commit-%: # target docker-compose-build: Fire docker-images-myos, Call docker-compose build SERVICE .PHONY: docker-compose-build docker-compose-build: docker-images-myos - $(call docker-compose,build $(if $(filter $(DOCKER_BUILD_NO_CACHE),true),--pull --no-cache) $(if $(filter $(SERVICE),$(SERVICES)),$(SERVICE))) + $(call docker-compose,build $(DOCKER_BUILD_ARGS) $(if $(filter $(SERVICE),$(SERVICES)),$(SERVICE))) # target docker-compose-config: Call docker-compose config .PHONY: docker-compose-config @@ -134,8 +134,9 @@ docker-network-create: docker-network-create-$(DOCKER_NETWORK) # target docker-network-create-%: Exec 'docker network create %' .PHONY: docker-network-create-% docker-network-create-%: - [ -n "$(shell docker network ls -q --filter name='^$*$$' 2>/dev/null)" ] \ - || { echo -n "Creating docker network $* ... " && $(RUN) docker network create $* >/dev/null 2>&1 && echo "done" || echo "ERROR"; } + if [ -z "$(shell docker network ls -q --filter name='^$*$$' 2>/dev/null)" ]; then \ + $(RUN) docker network create $* >/dev/null \ + && $(call INFO,docker network $* created); fi \ # target docker-network-rm: Fire docker-network-rm-% for DOCKER_NETWORK .PHONY: docker-network-rm @@ -144,14 +145,15 @@ docker-network-rm: docker-network-rm-$(DOCKER_NETWORK) # target docker-network-rm-%: Remove docker network % .PHONY: docker-network-rm-% docker-network-rm-%: - [ -z "$(shell docker network ls -q --filter name='^$*$$' 2>/dev/null)" ] \ - || { echo -n "Removing docker network $* ... " && $(RUN) docker network rm $* >/dev/null 2>&1 && echo "done" || echo "ERROR"; } + if [ -n "$(shell docker network ls -q --filter name='^$*$$' 2>/dev/null)" ]; then \ + $(RUN) docker network rm $* >/dev/null \ + && $(call INFO,docker network $* removed); fi \ # target docker-plugin-install: Exec 'docker plugin install DOCKER_PLUGIN_OPTIONS DOCKER_PLUGIN' .PHONY: docker-plugin-install docker-plugin-install: $(eval docker_plugin_state := $(shell docker plugin ls | awk '$$2 == "$(DOCKER_PLUGIN)" {print $$NF}') ) - $(if $(docker_plugin_state),$(if $(filter $(docker_plugin_state),false),echo -n "Enabling docker plugin $(DOCKER_PLUGIN) ... " && $(RUN) docker plugin enable $(DOCKER_PLUGIN) >/dev/null 2>&1 && echo "done" || echo "ERROR"),echo -n "Installing docker plugin $(DOCKER_PLUGIN) ... " && $(RUN) docker plugin install $(DOCKER_PLUGIN_OPTIONS) $(DOCKER_PLUGIN) $(DOCKER_PLUGIN_ARGS) >/dev/null 2>&1 && echo "done" || echo "ERROR") + $(if $(docker_plugin_state),$(if $(filter $(docker_plugin_state),false),printf "Enabling docker plugin $(DOCKER_PLUGIN) ... " && $(RUN) docker plugin enable $(DOCKER_PLUGIN) >/dev/null 2>&1 && printf "done\n" || printf "ERROR\n"),printf "Installing docker plugin $(DOCKER_PLUGIN) ... " && $(RUN) docker plugin install $(DOCKER_PLUGIN_OPTIONS) $(DOCKER_PLUGIN) $(DOCKER_PLUGIN_ARGS) >/dev/null 2>&1 && printf "done\n" || printf "ERROR\n") # target docker-push: Call docker-push for each SERVICES .PHONY: docker-push @@ -159,7 +161,7 @@ docker-push: ifneq ($(filter $(DEPLOY),true),) $(foreach service,$(or $(SERVICE),$(SERVICES)),$(call docker-push,$(service))) else - printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}target${COLOR_RESET} $@ ${COLOR_GREEN}not enabled in${COLOR_RESET} $(APP).\n" >&2 + $(call WARNING,disabled target,$@,$(APP)) endif # target docker-push-%: Call docker-push with tag % for each SERVICES @@ -168,7 +170,7 @@ docker-push-%: ifneq ($(filter $(DEPLOY),true),) $(foreach service,$(or $(SERVICE),$(SERVICES)),$(call docker-push,$(service),,$*)) else - printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}target${COLOR_RESET} $@ ${COLOR_GREEN}not enabled in${COLOR_RESET} $(APP).\n" >&2 + $(call WARNING,disabled target,$@,$(APP)) endif # target docker-rebuild: Call docker-build target with DOCKER_BUILD_CAHE=false @@ -203,7 +205,7 @@ docker-run-%: docker-build-% $(eval path := $(patsubst %/,%,$*)) $(eval image := $(DOCKER_REPOSITORY)/$(lastword $(subst /, ,$(path)))$(if $(findstring :,$*),,:$(DOCKER_IMAGE_TAG))) $(eval image_id := $(shell docker images -q $(image) 2>/dev/null)) - $(call docker-run,$(if $(image_id),$(image),$(path)),$(command)) + $(call docker-run,$(command),$(if $(image_id),$(image),$(path))) # target docker-tag: Call docker-tag for each SERVICES .PHONY: docker-tag @@ -211,7 +213,7 @@ docker-tag: ifneq ($(filter $(DEPLOY),true),) $(foreach service,$(or $(SERVICE),$(SERVICES)),$(call docker-tag,$(service))) else - printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}target${COLOR_RESET} $@ ${COLOR_GREEN}not enabled in${COLOR_RESET} $(APP).\n" >&2 + $(call WARNING,disabled target,$@,$(APP)) endif # target docker-tag-%: Call docker-tag with target tag % for each SERVICES @@ -220,7 +222,7 @@ docker-tag-%: ifneq ($(filter $(DEPLOY),true),) $(foreach service,$(or $(SERVICE),$(SERVICES)),$(call docker-tag,$(service),,,,$*)) else - printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}target${COLOR_RESET} $@ ${COLOR_GREEN}not enabled in${COLOR_RESET} $(APP).\n" >&2 + $(call WARNING,disabled target,$@,$(APP)) endif # target docker-volume-rm: Fire docker-volume-rm-% for COMPOSE_PROJECT_NAME diff --git a/make/apps/git.mk b/make/apps/git.mk index 402718f..3c321df 100644 --- a/make/apps/git.mk +++ b/make/apps/git.mk @@ -4,65 +4,77 @@ # target git-branch-create-upstream-%: Create git BRANCH from upstream/% branch .PHONY: git-branch-create-upstream-% git-branch-create-upstream-%: myos-base update-upstream - $(call exec,git fetch --prune upstream) - $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && echo Unable to create $(BRANCH). || git branch $(BRANCH) upstream/$*) - $(call exec,[ $$(git ls-remote --heads upstream $(BRANCH) |wc -l) -eq 0 ] && git push upstream $(BRANCH) || echo Unable to create branch $(BRANCH) on remote upstream.) - $(call exec,git checkout $(BRANCH)) + $(RUN) $(call exec,git fetch --prune upstream) + $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1) \ + && $(call WARNING,present branch,$(BRANCH)) \ + || $(RUN) $(call exec,git branch $(BRANCH) upstream/$*) + $(call exec,[ $$(git ls-remote --heads upstream $(BRANCH) |wc -l) -eq 0 ]) \ + && $(RUN) $(call exec,git push upstream $(BRANCH)) \ + || $(call WARNING,present branch,$(BRANCH),upstream) + $(RUN) $(call exec,git checkout $(BRANCH)) # target git-branch-delete: Delete git BRANCH .PHONY: git-branch-delete git-branch-delete: myos-base update-upstream - $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && git branch -d $(BRANCH) || echo Unable to delete branch $(BRANCH).) - $(foreach remote,upstream, $(call exec,[ $$(git ls-remote --heads $(remote) $(BRANCH) |wc -l) -eq 1 ] && git push $(remote) :$(BRANCH) || echo Unable to delete branch $(BRANCH) on remote $(remote).) &&) true + $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1) \ + && $(RUN) $(call exec,git branch -d $(BRANCH)) \ + || $(call WARNING,no branch,$(BRANCH)) + $(foreach remote,upstream,$(call exec,[ $$(git ls-remote --heads $(remote) $(BRANCH) |wc -l) -eq 1 ]) \ + && $(RUN) $(call exec,git push $(remote) :$(BRANCH)) \ + || $(call WARNING,no branch,$(BRANCH),$(remote)) \ + &&) true # target git-branch-merge-upstream-%: Merge git BRANCH into upstream/% branch .PHONY: git-branch-merge-upstream-% git-branch-merge-upstream-%: myos-base update-upstream - $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1) - $(call exec,git checkout $(BRANCH)) - $(call exec,git pull --ff-only upstream $(BRANCH)) - $(call exec,git push upstream $(BRANCH)) - $(call exec,git checkout $*) - $(call exec,git pull --ff-only upstream $*) - $(call exec,git merge --no-ff --no-edit $(BRANCH)) - $(call exec,git push upstream $*) + $(RUN) $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1) + $(RUN) $(call exec,git checkout $(BRANCH)) + $(RUN) $(call exec,git pull --ff-only upstream $(BRANCH)) + $(RUN) $(call exec,git push upstream $(BRANCH)) + $(RUN) $(call exec,git checkout $*) + $(RUN) $(call exec,git pull --ff-only upstream $*) + $(RUN) $(call exec,git merge --no-ff --no-edit $(BRANCH)) + $(RUN) $(call exec,git push upstream $*) # target git-stash: git stash .PHONY: git-stash git-stash: myos-base git-status if [ ! $(STATUS) -eq 0 ]; then \ - $(call exec,git stash); \ + $(RUN) $(call exec,git stash); \ fi # target git-status: Define STATUS with number of lines of git status .PHONY: git-status git-status: myos-base $(eval IGNORE_DRYRUN := true) - $(eval STATUS := $(shell $(call exec,git status -uno --porcelain 2>/dev/null |wc -l))) + $(eval STATUS := $(shell $(call exec,sh -c 'git status -uno --porcelain 2>/dev/null |wc -l'))) $(eval IGNORE_DRYRUN := false) # target git-tag-create-upstream-%: Create git TAG to reference upstream/% branch .PHONY: git-tag-create-upstream-% git-tag-create-upstream-%: myos-base update-upstream ifneq ($(words $(TAG)),0) - $(call exec,git checkout $*) - $(call exec,git pull --tags --prune upstream $*) - $(call sed,s/^##\? $(TAG).*/## $(TAG) - $(shell date +%Y-%m-%d)/,CHANGELOG.md) - $(call exec,[ $$(git diff CHANGELOG.md 2>/dev/null |wc -l) -eq 0 ] || git commit -m "$$(cat CHANGELOG.md |sed -n '\''/$(TAG)/,/^$$/{s/##\(.*\)/release\1\n/;p;}'\'')" CHANGELOG.md) - $(call exec,[ $$(git tag -l $(TAG) |wc -l) -eq 0 ] || git tag -d $(TAG)) - $(call exec,git tag $(TAG)) - $(call exec,[ $$(git ls-remote --tags upstream $(TAG) |wc -l) -eq 0 ] || git push upstream :refs/tags/$(TAG)) - $(call exec,git push --tags upstream $*) + $(RUN) $(call exec,git checkout $*) + $(RUN) $(call exec,git pull --tags --prune upstream $*) + $(RUN) $(call sed,s/^##\? $(TAG).*/## $(TAG) - $(shell date +%Y-%m-%d)/,CHANGELOG.md) + $(call exec,[ $$(git diff CHANGELOG.md 2>/dev/null |wc -l) -eq 0 ]) \ + || $(RUN) $(call exec,git commit -m "$$(cat CHANGELOG.md |sed -n '/$(TAG)/,/^$$/{s/##\(.*\)/release\1\n/;p;}')" CHANGELOG.md) + $(call exec,[ $$(git tag -l $(TAG) |wc -l) -eq 0 ]) \ + || $(RUN) $(call exec,git tag -d $(TAG)) + $(RUN) $(call exec,git tag $(TAG)) + $(call exec,[ $$(git ls-remote --tags upstream $(TAG) |wc -l) -eq 0 ]) \ + || $(RUN) $(call exec,git push upstream :refs/tags/$(TAG)) + $(RUN) $(call exec,git push --tags upstream $*) endif # target git-tag-merge-upstream-%: Merge git TAG into upstream/% branch .PHONY: git-tag-merge-upstream-% git-tag-merge-upstream-%: myos-base update-upstream ifneq ($(words $(TAG)),0) - $(call exec,git fetch --tags -u --prune upstream $*:$*) - $(call exec,git checkout $*) - $(call exec,git merge --ff --no-edit $(TAG)) - $(call exec,git push upstream $*) + $(RUN) $(call exec,git fetch --tags -u --prune upstream $*:$*) + $(RUN) $(call exec,git checkout $*) + $(RUN) $(call exec,git merge --ff --no-edit $(TAG)) + $(RUN) $(call exec,git push upstream $*) endif # target git-unstash: git stash pop @@ -70,5 +82,5 @@ endif git-unstash: myos-base $(eval STATUS ?= 0) if [ ! $(STATUS) -eq 0 ]; then \ - $(call exec,git stash pop); \ + $(RUN) $(call exec,git stash pop); \ fi diff --git a/make/apps/install.mk b/make/apps/install.mk index d71c0d6..89d41de 100644 --- a/make/apps/install.mk +++ b/make/apps/install.mk @@ -7,10 +7,14 @@ ## it creates user % with password % and all privileges on database % ## it imports %.mysql.gz file in database % .PHONY: install-mysql-database-% -install-mysql-database-%: myos-base - $(call exec,mysql -h mysql -u root -proot $* -e "use $*" >/dev/null 2>&1 || mysql -h mysql -u root -proot mysql -e "create database $* character set utf8 collate utf8_unicode_ci;") - $(call exec,mysql -h mysql -u $* -p$* $* -e "use $*" >/dev/null 2>&1 || mysql -h mysql -u root -proot mysql -e "grant all privileges on $*.* to '\''$*'\''@'\''%'\'' identified by '\''$*'\''; flush privileges;") - $(call exec,[ $$(mysql -h mysql -u $* -p$* $* -e "show tables" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.mysql.gz" ] && gzip -cd "${APP_DIR}/$*.mysql.gz" |mysql -h mysql -u root -proot $* || true) +install-mysql-database-%: $(if $(DOCKER_RUN),myos-base) + $(call exec,mysql -h mysql -u root -proot $* -e "use $*" >/dev/null 2>&1) \ + || $(RUN) $(call exec,mysql -h mysql -u root -proot mysql -e "create database $* character set utf8 collate utf8_unicode_ci;") + $(call exec,mysql -h mysql -u $* -p$* $* -e "use $*" >/dev/null 2>&1) \ + || $(RUN) $(call exec,mysql -h mysql -u root -proot mysql -e "grant all privileges on $*.* to '$*'@'%' identified by '$*'; flush privileges;") + $(call exec,sh -c '[ $$(mysql -h mysql -u $* -p$* $* -e "show tables" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.mysql.gz" ]') \ + && $(RUN) $(call exec,sh -c 'gzip -cd "${APP_DIR}/$*.mysql.gz" |mysql -h mysql -u root -proot $*') \ + ||: # target install-pgsql-database-%: Import %.pgsql.gz to database % # on local host @@ -19,10 +23,16 @@ install-mysql-database-%: myos-base ## it imports %.pgsql.gz file in database % .PHONY: install-pgsql-database-% install-pgsql-database-%: myos-base - $(call exec,PGPASSWORD=$* psql -h postgres -U $* template1 -c "\q" >/dev/null 2>&1 || PGPASSWORD=postgres psql -h postgres -U postgres -c "create user $* with createdb password '\''$*'\'';") - $(call exec,PGPASSWORD=$* psql -h postgres -U $* -d $* -c "" >/dev/null 2>&1 || PGPASSWORD=postgres psql -h postgres -U postgres -c "create database $* owner $* ;") - $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql.gz" ] && gzip -cd "${APP_DIR}/$*.pgsql.gz" |PGPASSWORD="postgres" psql -h postgres -U postgres -d $* || true) - $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql" ] && PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH SUPERUSER" && PGPASSWORD="postgres" pg_restore -h postgres --no-owner --role=$* -U postgres -d $* ${APP_DIR}/$*.pgsql && PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH NOSUPERUSER" || true) + $(call exec,PGPASSWORD=$* psql -h postgres -U $* template1 -c "\q" >/dev/null 2>&1) \ + || $(RUN) $(call exec,PGPASSWORD=postgres psql -h postgres -U postgres -c "create user $* with createdb password '$*';") + $(call exec,PGPASSWORD=$* psql -h postgres -U $* -d $* -c "" >/dev/null 2>&1) \ + || $(RUN) $(call exec,PGPASSWORD=postgres psql -h postgres -U postgres -c "create database $* owner $* ;") + $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql.gz" ]) \ + && $(RUN) $(call exec,sh -c 'gzip -cd "${APP_DIR}/$*.pgsql.gz" |PGPASSWORD="postgres" psql -h postgres -U postgres -d $*') \ + ||: + $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql" ]) \ + && $(RUN) $(call exec,sh -c 'PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH SUPERUSER" && PGPASSWORD="postgres" pg_restore -h postgres --no-owner --role=$* -U postgres -d $* ${APP_DIR}/$*.pgsql && PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH NOSUPERUSER"') \ + ||: # target install-build-config: Call install-config with file * and dest build .PHONY: install-build-config diff --git a/make/apps/myos/def.ssh.mk b/make/apps/myos/def.ssh.mk index 02e99f8..d5a8a8d 100644 --- a/make/apps/myos/def.ssh.mk +++ b/make/apps/myos/def.ssh.mk @@ -14,7 +14,7 @@ define ssh-connect $(eval command := $(2)) $(eval user := $(or $(3),root)) $(eval DOCKER_EXEC_OPTIONS := -it) - $(foreach host,$(hosts),$(call exec,ssh -t -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $(user)@$(host) "$(command)") ||) true + $(foreach host,$(hosts),$(RUN) $(call exec,ssh -t -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $(user)@$(host) "$(command)") ||) true endef # function ssh-exec: Exec command 2 on remote hosts 1 without tty @@ -23,5 +23,5 @@ define ssh-exec $(eval hosts := $(1)) $(eval command := $(2)) $(eval user := $(or $(3),root)) - $(foreach host,$(hosts),$(call exec,ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $(user)@$(host) "$(command)") &&) true + $(foreach host,$(hosts),$(RUN) $(call exec,ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $(user)@$(host) "$(command)") &&) true endef diff --git a/make/apps/myos/ssh.mk b/make/apps/myos/ssh.mk index 1f06075..76523eb 100644 --- a/make/apps/myos/ssh.mk +++ b/make/apps/myos/ssh.mk @@ -1,26 +1,59 @@ ## # SSH -# target get-PrivateIpAddress-%: Fire aws-ec2-get-instances-PrivateIpAddress-% -.PHONY: get-PrivateIpAddress-% -get-PrivateIpAddress-%: aws-ec2-get-instances-PrivateIpAddress-%; - # target ssh: Call ssh-connect ARGS or SHELL .PHONY: ssh -ssh: get-PrivateIpAddress-$(SERVER_NAME) ## Connect to first remote host +ssh: ssh-get-PrivateIpAddress-$(SERVER_NAME) ## Connect to first remote host $(call ssh-connect,$(AWS_INSTANCE_IP),$(if $(ARGS),$(ARGS),$(SHELL))) +# target ssh-add: Fire ssh-key and ssh-add file SSH_PRIVATE_KEYS in folder SSH_DIR +.PHONY: ssh-add +ssh-add: ssh-key + $(eval SSH_PRIVATE_KEYS := $(foreach file,$(SSH_DIR)/id_rsa $(filter-out $(wildcard $(SSH_DIR)/id_rsa),$(wildcard $(SSH_DIR)/*)),$(if $(shell grep "PRIVATE KEY" $(file) 2>/dev/null),$(notdir $(file))))) + $(call run,sh -c '$(foreach file,$(patsubst %,$(SSH_DIR)/%,$(SSH_PRIVATE_KEYS)),ssh-add -l |grep -qw $$(ssh-keygen -lf $(file) 2>/dev/null |awk '\''{print $$2}'\'') 2>/dev/null || $(RUN) ssh-add $(file) ||: &&) true',-v $(SSH_DIR):$(SSH_DIR) $(DOCKER_IMAGE_CLI) ) + # target ssh-connect: Call ssh-connect make connect SERVICE .PHONY: ssh-connect -ssh-connect: get-PrivateIpAddress-$(SERVER_NAME) - $(call ssh-connect,$(AWS_INSTANCE_IP),make connect $(if $(SERVICE),SERVICE=$(SERVICE))) +ssh-connect: ssh-get-PrivateIpAddress-$(SERVER_NAME) + $(call ssh-connect,$(AWS_INSTANCE_IP),make connect COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) ENV=$(ENV) $(if $(SERVICE),SERVICE=$(SERVICE))) + +# target ssh-del: ssh-add -d file SSH_PRIVATE_KEYS in folder SSH_DIR +.PHONY: ssh-del +ssh-del: + $(eval SSH_PRIVATE_KEYS := $(foreach file,$(SSH_DIR)/id_rsa $(filter-out $(wildcard $(SSH_DIR)/id_rsa),$(wildcard $(SSH_DIR)/*)),$(if $(shell grep "PRIVATE KEY" $(file) 2>/dev/null),$(notdir $(file))))) + $(call run,sh -c '$(foreach file,$(patsubst %,$(SSH_DIR)/%,$(SSH_PRIVATE_KEYS)),ssh-add -l |grep -qw $$(ssh-keygen -lf $(file) 2>/dev/null |awk '\''{print $$2}'\'') 2>/dev/null && $(RUN) ssh-add -d $(file) ||: &&) true',-v $(SSH_DIR):$(SSH_DIR) $(DOCKER_IMAGE_CLI) ) # target ssh-exec: Call ssh-exec make exec SERVICE ARGS .PHONY: ssh-exec -ssh-exec: get-PrivateIpAddress-$(SERVER_NAME) - $(call ssh-exec,$(AWS_INSTANCE_IP),make exec $(if $(SERVICE),SERVICE=$(SERVICE)) $(if $(ARGS),ARGS='\''"$(ARGS)"'\'')) +ssh-exec: ssh-get-PrivateIpAddress-$(SERVER_NAME) + $(call ssh-exec,$(AWS_INSTANCE_IP),make exec COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) ENV=$(ENV) $(if $(SERVICE),SERVICE=$(SERVICE)) $(if $(ARGS),ARGS='\''"$(ARGS)"'\'')) + +# target ssh-get-PrivateIpAddress-%: Fire aws-ec2-get-instances-PrivateIpAddress-% +.PHONY: ssh-get-PrivateIpAddress-% +ssh-get-PrivateIpAddress-%: aws-ec2-get-instances-PrivateIpAddress-%; + +# target ssh-key: Add ssh private key SSH_KEY to SSH_DIR +.PHONY: ssh-key +ssh-key: $(if $(DOCKER_RUN),stack-base-up) +ifneq (,$(filter true,$(DRONE))) + $(call exec,sh -c '[ ! -d $(SSH_DIR) ] && mkdir -p $(SSH_DIR) && chown $(UID) $(SSH_DIR) && chmod 0700 $(SSH_DIR) ||:') +endif + $(if $(SSH_KEY),$(eval export SSH_KEY ?= $(SSH_KEY)) $(call env-run,sh -c 'printf "$$SSH_KEY\n" > $(SSH_DIR)/$(COMPOSE_PROJECT_NAME)_id_rsa && chmod 0600 $(SSH_DIR)/$(COMPOSE_PROJECT_NAME)_id_rsa && chown $(UID) $(SSH_DIR)/$(COMPOSE_PROJECT_NAME)_id_rsa ||:')) # target ssh-run: Call ssh-run make run SERVICE ARGS .PHONY: ssh-run -ssh-run: get-PrivateIpAddress-$(SERVER_NAME) +ssh-run: ssh-get-PrivateIpAddress-$(SERVER_NAME) $(call ssh-exec,$(AWS_INSTANCE_IP),make run $(if $(SERVICE),SERVICE=$(SERVICE)) $(if $(ARGS),ARGS='\''"$(ARGS)"'\'')) + +# target ssl-certs: Create ${DOMAIN}.key.pem and ${DOMAIN}.crt.pem files +.PHONY: ssl-certs +ssl-certs: + docker run --rm --mount source=$(COMPOSE_PROJECT_NAME_NODE)_ssl-certs,target=/certs alpine [ -f /certs/$(DOMAIN).crt.pem -a -f /certs/$(DOMAIN).key.pem ] \ + || $(RUN) docker run --rm -e DOMAIN=$(DOMAIN) --mount source=$(COMPOSE_PROJECT_NAME_NODE)_ssl-certs,target=/certs alpine sh -c "\ + apk --no-cache add openssl \ + && { [ -f /certs/${DOMAIN}.key.pem ] || openssl genrsa -out /certs/${DOMAIN}.key.pem 2048; } \ + && openssl req -key /certs/${DOMAIN}.key.pem -out /certs/${DOMAIN}.crt.pem \ + -addext extendedKeyUsage=serverAuth \ + -addext subjectAltName=DNS:${DOMAIN} \ + -subj \"/C=/ST=/L=/O=/CN=${DOMAIN}\" \ + -x509 -days 365" diff --git a/make/apps/release.mk b/make/apps/release.mk index 45d0870..a86816e 100644 --- a/make/apps/release.mk +++ b/make/apps/release.mk @@ -43,4 +43,4 @@ release-upgrade: $(patsubst %,release-upgrade-from-%,$(RELEASE_UPGRADE)) release # target release-upgrade-from-%: Sample of catch-all release migration target .PHONY: release-upgrade-from-% release-upgrade-from-%: - echo "Upgrading from release: $*" + printf "Upgrading from release: $*\n" diff --git a/make/apps/setup.mk b/make/apps/setup.mk index 999af12..98bfa34 100644 --- a/make/apps/setup.mk +++ b/make/apps/setup.mk @@ -4,7 +4,7 @@ .PHONY: setup-sysctl setup-sysctl: ifeq ($(SETUP_SYSCTL),true) - $(foreach config,$(SETUP_SYSCTL_CONFIG),$(call docker-run,--privileged alpine:latest,sysctl -q -w $(config)) &&) true + $(foreach config,$(SETUP_SYSCTL_CONFIG),$(call docker-run,sysctl -q -w $(config),--privileged alpine) &&) true endif .PHONY: setup-nfsd @@ -20,8 +20,8 @@ define setup-nfsd-osx $(eval dir:=$(or $(1),$(MONOREPO_DIR))) $(eval uid:=$(or $(2),$(UID))) $(eval gid:=$(or $(3),$(GID))) - grep "$(dir)" /etc/exports >/dev/null 2>&1 || echo "$(dir) -alldirs -mapall=$(uid):$(gid) localhost" |sudo tee -a /etc/exports >/dev/null - $(foreach config,$(SETUP_NFSD_OSX_CONFIG),grep "$(config)" /etc/nfs.conf >/dev/null 2>&1 || echo "$(config)" |sudo tee -a /etc/nfs.conf >/dev/null &&) true + grep "$(dir)" /etc/exports >/dev/null 2>&1 || printf "$(dir) -alldirs -mapall=$(uid):$(gid) localhost\n" |sudo tee -a /etc/exports >/dev/null + $(foreach config,$(SETUP_NFSD_OSX_CONFIG),grep "$(config)" /etc/nfs.conf >/dev/null 2>&1 || printf "$(config)\n" |sudo tee -a /etc/nfs.conf >/dev/null &&) true nfsd status >/dev/null || sudo nfsd enable showmount -e localhost |grep "$(dir)" >/dev/null 2>&1 || sudo nfsd restart endef diff --git a/make/common.mk b/make/common.mk index b3cf7a3..c92baaf 100644 --- a/make/common.mk +++ b/make/common.mk @@ -46,24 +46,24 @@ update-config: myos-base ## it reads .env files to extract applications hostnames and add it to /etc/hosts .PHONY: update-hosts update-hosts: -ifneq (,$(filter $(ENV),master)) - cat */.env 2>/dev/null |grep -Eo 'urlprefix-[^/]+' |sed 's/urlprefix-//' |while read host; do grep $$host /etc/hosts >/dev/null 2>&1 || { echo "Adding $$host to /etc/hosts"; echo 127.0.0.1 $$host |$(RUN) sudo tee -a /etc/hosts >/dev/null; }; done +ifneq (,$(filter $(ENV),local)) + cat .env */.env 2>/dev/null |grep -Eo 'urlprefix-[^/]+' |sed 's/urlprefix-//' |while read host; do grep $$host /etc/hosts >/dev/null 2>&1 || { printf "Adding $$host to /etc/hosts\n"; printf "127.0.0.1 $$host\n" |$(RUN) sudo tee -a /etc/hosts >/dev/null; }; done endif # target update-remote-%: fetch git remote % .PHONY: update-remote-% update-remote-%: myos-base - $(call exec,git fetch --prune --tags $*) + $(RUN) $(call exec,git fetch --prune --tags $*) # target update-remotes: fetch all git remotes .PHONY: update-remotes update-remotes: myos-base - $(call exec,git fetch --all --prune --tags) + $(RUN) $(call exec,git fetch --all --prune --tags) # target update-upstream: fetch git remote upstream .PHONY: update-upstream update-upstream: myos-base .git/refs/remotes/upstream/master - $(call exec,git fetch --prune --tags upstream) + $(RUN) $(call exec,git fetch --prune --tags upstream) # target .git/refs/remotes/upstream/master: git add upstream APP_UPSTREAM_REPOSITORY .git/refs/remotes/upstream/master: diff --git a/make/def.docker.mk b/make/def.docker.mk index 8bbaaf6..f29e9a2 100644 --- a/make/def.docker.mk +++ b/make/def.docker.mk @@ -1,6 +1,6 @@ -COMPOSE_VERSION ?= 1.24.1 -COMPOSE_PROJECT_NAME_MYOS ?= $(USER)_$(ENV)_myos +COMPOSE_PROJECT_NAME_MYOS ?= $(USER_ENV)_myos COMPOSE_PROJECT_NAME_NODE ?= node +COMPOSE_VERSION ?= 1.29.2 DOCKER_ENV ?= $(env.docker) DOCKER_EXEC_OPTIONS ?= DOCKER_IMAGE ?= $(DOCKER_IMAGE_CLI) @@ -10,15 +10,16 @@ DOCKER_NAME ?= $(DOCKER_NAME_CLI) DOCKER_NAME_CLI ?= $(COMPOSE_PROJECT_NAME_MYOS)_cli DOCKER_NAME_SSH ?= $(COMPOSE_PROJECT_NAME_MYOS)_ssh DOCKER_NETWORK ?= $(DOCKER_NETWORK_PRIVATE) -DOCKER_NETWORK_PRIVATE ?= $(USER)_$(ENV) +DOCKER_NETWORK_PRIVATE ?= $(USER_ENV) DOCKER_NETWORK_PUBLIC ?= node DOCKER_REPOSITORY_MYOS ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_MYOS)) DOCKER_REPOSITORY_NODE ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_NODE)) -# DOCKER_RUN_OPTIONS: default options to `docker run` command -DOCKER_RUN_OPTIONS ?= --rm -it -# DOCKER_RUN_VOLUME: options to `docker run` command to mount additionnal volumes -DOCKER_RUN_VOLUME ?= -v $$PWD:$$PWD -DOCKER_RUN_WORKDIR ?= -w $$PWD +DOCKER_RUN ?= $(filter true,$(DOCKER)) +# DOCKER_RUN_OPTIONS: default options of `docker run` command +DOCKER_RUN_OPTIONS += --rm -it +# DOCKER_RUN_VOLUME: options -v of `docker run` command to mount additionnal volumes +DOCKER_RUN_VOLUME += -v /var/run/docker.sock:/var/run/docker.sock +DOCKER_RUN_WORKDIR ?= -w $(PWD) DOCKER_VOLUME_SSH ?= $(COMPOSE_PROJECT_NAME_MYOS)_ssh ENV_VARS += DOCKER_NETWORK_PRIVATE DOCKER_NETWORK_PUBLIC DOCKER_REPOSITORY_MYOS DOCKER_REPOSITORY_NODE DOCKER_VOLUME_SSH @@ -27,64 +28,54 @@ DOCKER_RUN_OPTIONS := --rm --network $(DOCKER_NETWORK) # When running docker command in drone, we are already in a docker (dind). # Whe need to find the volume mounted in the current docker (runned by drone) to mount it in our docker command. # If we do not mount the volume in our docker, we wont be able to access the files in this volume as the /drone/src directory would be empty. -DOCKER_RUN_VOLUME := -v /var/run/docker.sock:/var/run/docker.sock -v $$(docker inspect $$(basename $$(cat /proc/1/cpuset)) 2>/dev/null |awk 'BEGIN {FS=":"} $$0 ~ /"drone-[a-zA-Z0-9]*:\/drone"$$/ {gsub(/^[ \t\r\n]*"/,"",$$1); print $$1; exit}'):/drone $(if $(wildcard /root/.netrc),-v /root/.netrc:/root/.netrc) +DOCKER_RUN_VOLUME += -v $$(docker inspect $$(basename $$(cat /proc/1/cpuset)) 2>/dev/null |awk 'BEGIN {FS=":"} $$0 ~ /"drone-[a-zA-Z0-9]*:\/drone"$$/ {gsub(/^[ \t\r\n]*"/,"",$$1); print $$1; exit}'):/drone $(if $(wildcard /root/.netrc),-v /root/.netrc:/root/.netrc) else -DOCKER_RUN_VOLUME := -v /var/run/docker.sock:/var/run/docker.sock -v $(or $(MONOREPO_DIR),$(APP_DIR)):$(or $(WORKSPACE_DIR),$(MONOREPO_DIR),$(APP_DIR)) +DOCKER_RUN_VOLUME += -v $(or $(APP_PARENT_DIR),$(APP_DIR),$(PWD)):$(or $(WORKSPACE_DIR),$(APP_PARENT_DIR),$(APP_DIR),$(PWD)) endif -# function env-run: Call env-exec with arg 1 in a subshell -define env-run - $(call INFO,env-run,$(1)) - $(call env-exec,sh -c '$(or $(1),$(SHELL))') -endef -# function env-exec: Exec arg 1 in a new env -define env-exec - $(call INFO,env-exec,$(1)) - IFS=$$'\n'; env $(env_reset) $(env) $(1) -endef - ifeq ($(DOCKER), true) DOCKER_SSH_AUTH := -e SSH_AUTH_SOCK=/tmp/ssh-agent/socket -v $(DOCKER_VOLUME_SSH):/tmp/ssh-agent -# function docker-run: Run new DOCKER_IMAGE:DOCKER_IMAGE_TAG docker with arg 2 +# function docker-run: Run docker image 2 with arg 1 define docker-run $(call INFO,docker-run,$(1)$(comma) $(2)) - $(call run,$(or $(1),$(DOCKER_IMAGE):$(DOCKER_IMAGE_TAG)) $(2)) + $(call run,$(or $(2),$(DOCKER_IMAGE)) $(1)) endef ifeq ($(DRONE), true) -# function exec: Run new DOCKER_IMAGE docker with arg 1 +# function exec DRONE=true: Run DOCKER_IMAGE with arg 1 define exec $(call INFO,exec,$(1)) - $(call run,$(DOCKER_IMAGE) sh -c '$(or $(1),$(SHELL))') + $(call run,$(DOCKER_IMAGE) $(or $(1),$(SHELL))) endef else # function exec: Exec arg 1 in docker DOCKER_NAME define exec $(call INFO,exec,$(1)) - $(RUN) docker exec $(DOCKER_ENV) $(DOCKER_EXEC_OPTIONS) $(DOCKER_RUN_WORKDIR) $(DOCKER_NAME) sh -c '$(or $(1),$(SHELL))' + $(RUN) docker exec $(DOCKER_ENV) $(DOCKER_EXEC_OPTIONS) $(DOCKER_RUN_WORKDIR) $(DOCKER_NAME) $(or $(1),$(SHELL)) endef endif -# function run: Pass arg 1 to docker run +# function run: Run docker run with arg 1 and docker repository 2 +## attention: arg 2 should end with slash or space define run - $(call INFO,run,$(1)) - $(RUN) docker run $(DOCKER_ENV) $(DOCKER_RUN_OPTIONS) $(DOCKER_RUN_VOLUME) $(DOCKER_RUN_WORKDIR) $(DOCKER_SSH_AUTH) $(1) + $(call INFO,run,$(1)$(comma) $(2)) + $(RUN) docker run $(DOCKER_ENV) $(DOCKER_RUN_OPTIONS) $(DOCKER_RUN_VOLUME) $(DOCKER_RUN_WORKDIR) $(DOCKER_SSH_AUTH) $(2)$(1) endef else SHELL := /bin/bash -# function docker-run: Run new DOCKER_IMAGE:DOCKER_IMAGE_TAG docker with arg 2 +# function docker-run DOCKER=false: Run docker image 2 with arg 1 define docker-run $(call INFO,docker-run,$(1)$(comma) $(2)) - $(RUN) docker run $(DOCKER_RUN_OPTIONS) $(DOCKER_ENV) $(DOCKER_RUN_VOLUME) $(DOCKER_RUN_WORKDIR) $(or $(1),$(DOCKER_IMAGE):$(DOCKER_IMAGE_TAG)) $(2) + $(RUN) docker run $(DOCKER_ENV) $(DOCKER_RUN_OPTIONS) $(DOCKER_RUN_VOLUME) $(DOCKER_RUN_WORKDIR) $(or $(2),$(DOCKER_IMAGE)) $(1) endef -# function exec: Call env-exec with arg 1 or SHELL +# function exec DOCKER=false: Call env-exec with arg 1 or SHELL define exec $(call INFO,exec,$(1)) $(call env-exec,$(or $(1),$(SHELL))) endef -# function run: Call env-run with arg 1 +# function run DOCKER=false: Call env-run with arg 1 define run $(call INFO,run,$(1)) $(call env-run,$(1)) @@ -101,3 +92,15 @@ define docker-volume-copy $(RUN) docker volume inspect $(to) >/dev/null 2>&1 || $(RUN) docker volume create $(to) >/dev/null $(RUN) docker run --rm -v $(from):/from -v $(to):/to alpine ash -c "cd /from; cp -a . /to" endef + +# function env-run: Call env-exec with arg 1 +define env-run + $(call INFO,env-run,$(1)) + $(call env-exec,$(or $(1),$(SHELL))) +endef + +# function env-exec: Exec arg 1 with custom env +define env-exec + $(call INFO,env-exec,$(1)) + IFS=$$'\n'; env $(env_reset) $(env) $(1) +endef diff --git a/make/def.mk b/make/def.mk index b3db42a..b7431c9 100644 --- a/make/def.mk +++ b/make/def.mk @@ -1,3 +1,5 @@ +.DEFAULT_GOAL := help +.PHONY: FORCE comma ?= , dollar ?= $ dquote ?= " @@ -11,11 +13,24 @@ APPS ?= $(if $(MONOREPO),$(sort $(patsubst $(MONOREPO APPS_NAME ?= $(foreach app,$(APPS),$(or $(shell awk -F '=' '$$1 == "APP" {print $$2}' $(or $(wildcard $(MONOREPO_DIR)/$(app)/.env),$(wildcard $(MONOREPO_DIR)/$(app)/.env.$(ENV)),$(MONOREPO_DIR)/$(app)/.env.dist) 2>/dev/null),$(app))) BRANCH ?= $(GIT_BRANCH) CMDS ?= exec exec:% exec@% install-app install-apps run run:% run@% +COLOR_INFO ?= $(COLOR_BROWN) +COLOR_HIGHLIGHT ?= $(COLOR_GREEN) +COLOR_VALUE ?= $(COLOR_CYAN) +COLOR_WARNING ?= $(COLOR_YELLOW) +COLOR_RESET ?= \033[0m +COLOR_DGRAY ?= \033[30m +COLOR_RED ?= \033[31m +COLOR_GREEN ?= \033[32m +COLOR_BROWN ?= \033[33m +COLOR_YELLOW ?= \033[01;33m +COLOR_BLUE ?= \033[01;34m +COLOR_CYAN ?= \033[36m +COLOR_GRAY ?= \033[37m COMMIT ?= $(or $(SUBREPO_COMMIT),$(GIT_COMMIT)) CONFIG ?= $(RELATIVE)config CONFIG_REPOSITORY ?= $(call pop,$(or $(APP_UPSTREAM_REPOSITORY),$(GIT_UPSTREAM_REPOSITORY)))/$(notdir $(CONFIG)) CONTEXT ?= $(if $(APP),APP BRANCH DOMAIN VERSION) $(shell awk 'BEGIN {FS="="}; $$1 !~ /^(\#|$$)/ {print $$1}' .env.dist 2>/dev/null) -CONTEXT_DEBUG ?= MAKEFILE_LIST env env.docker APPS GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME LOG_LEVEL MAKE_DIR MAKE_SUBDIRS MAKE_CMD_ARGS MAKE_ENV_ARGS MONOREPO_DIR UID USER +CONTEXT_DEBUG ?= MAKEFILE_LIST env env.docker APPS GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME MAKE_DIR MAKE_SUBDIRS MAKE_CMD_ARGS MAKE_ENV_ARGS UID USER DEBUG ?= DOCKER ?= $(if $(BUILD),false,true) DOMAIN ?= localhost @@ -40,7 +55,7 @@ GIT_UPSTREAM_USER ?= $(lastword $(subst /, ,$(call pop,$(MYOS_REPO GIT_VERSION ?= $(shell git describe --tags $(BRANCH) 2>/dev/null || git rev-parse $(BRANCH) 2>/dev/null) HOSTNAME ?= $(shell hostname 2>/dev/null |sed 's/\..*//') IGNORE_DRYRUN ?= false -IGNORE_VERBOSE ?= $(IGNORE_DRYRUN) +IGNORE_VERBOSE ?= false LOG_LEVEL ?= $(if $(DEBUG),debug,$(if $(VERBOSE),info,error)) MAKE_ARGS ?= $(foreach var,$(MAKE_VARS),$(if $($(var)),$(var)='$($(var))')) MAKE_SUBDIRS ?= $(if $(filter myos,$(MYOS)),monorepo,$(if $(APP),apps $(foreach type,$(APP_TYPE),$(if $(wildcard $(MAKE_DIR)/apps/$(type)),apps/$(type))))) @@ -69,6 +84,7 @@ SUBREPO ?= $(if $(wildcard .gitrepo),$(notdir $(CURDIR)) TAG ?= $(GIT_TAG) UID ?= $(shell id -u 2>/dev/null) USER ?= $(shell id -nu 2>/dev/null) +USER_ENV ?= $(USER)_$(ENV) VERBOSE ?= $(if $(DEBUG),true) VERSION ?= $(GIT_VERSION) @@ -152,8 +168,23 @@ force = $$(while true; do [ $$(ps x |awk 'BEGIN {nargs=split("'"$$*"'",args)} $$ # macro gid: Return GID of group 1 gid = $(shell grep '^$(1):' /etc/group 2>/dev/null |awk -F: '{print $$3}') +INFO_FD := 2 # macro INFO: customized info -INFO = $(if $(VERBOSE),$(if $(filter-out true,$(IGNORE_VERBOSE)),printf '${COLOR_BROWN}$(APP)${COLOR_RESET}[${COLOR_GREEN}$(MAKELEVEL)${COLOR_RESET}] ${COLOR_BLUE}$@${COLOR_RESET}:${COLOR_RESET} ${COLOR_GREEN}Calling${COLOR_RESET} $(1)$(if $(2),$(lbracket)$(2)$(rbracket)) $(if $(3),${COLOR_BLUE}in folder${COLOR_RESET} $(3) )\n' >&2)) +INFO = \ +$(if $(VERBOSE),$(if $(filter-out true,$(IGNORE_VERBOSE)), \ + printf '${COLOR_INFO}$(APP)${COLOR_RESET}\ +[${COLOR_VALUE}$(MAKELEVEL)${COLOR_RESET}] \ +${COLOR_HIGHLIGHT}$@${COLOR_RESET}:${COLOR_RESET} ' >&$(INFO_FD) \ + $(if $(2), \ + && printf 'Call ${COLOR_HIGHLIGHT}$(1)${COLOR_RESET}$(lbracket)' >&$(INFO_FD) \ + && $(or $(strip $(call PRINTF,$(2))),printf '$(2)') >&$(INFO_FD) \ + && printf '$(rbracket)' >&$(INFO_FD) \ + $(if $(3),&& printf ' ${COLOR_VALUE}in${COLOR_RESET} $(3)' >&$(INFO_FD)) \ + , \ + && $(strip $(call PRINTF,$(1)) >&$(INFO_FD)) \ + ) \ + && printf '\n' >&$(INFO_FD) \ +)) # function install-app: Exec 'git clone url 1 dir 2' or Call update-app with url 1 dir 2 define install-app @@ -162,7 +193,7 @@ define install-app $(eval dir := $(or $(2), $(RELATIVE)$(lastword $(subst /, ,$(url))))) $(if $(wildcard $(dir)/.git), \ $(call update-app,$(url),$(dir)), \ - $(call exec,$(RUN) git clone $(QUIET) $(url) $(dir)) \ + $(RUN) $(call exec,git clone $(QUIET) $(url) $(dir)) \ ) endef @@ -196,7 +227,7 @@ endef pop = $(patsubst %$(or $(2),/)$(lastword $(subst $(or $(2),/), ,$(1))),%,$(1)) # macro sed: Exec sed script 1 on file 2 -sed = $(call exec,sed -i $(SED_SUFFIX) '\''$(1)'\'' $(2)) +sed = $(RUN) $(call exec,sed -i $(SED_SUFFIX) '$(1)' $(2)) # macro TIME: Print time elapsed since unixtime 1 TIME = awk '{printf "%02d:%02d:%02d\n",int($$1/3600),int(($$1%3600)/60),int($$1%60)}' \ @@ -208,7 +239,7 @@ define update-app $(eval url := $(or $(1), $(APP_REPOSITORY))) $(eval dir := $(or $(2), $(APP_DIR))) $(if $(wildcard $(dir)/.git), \ - $(call exec,cd $(dir) && $(RUN) git pull $(QUIET)), \ + $(RUN) $(call exec,sh -c 'cd $(dir) && git pull $(QUIET)'), \ $(call install-app,$(url),$(dir)) \ ) endef @@ -223,6 +254,20 @@ $(TARGET): $$(call make,$$*,,ENV_FILE) endef +WARNING_FD := 2 +# macro WARNING: customized warning +WARNING = printf '${COLOR_WARNING}WARNING:${COLOR_RESET} ${COLOR_INFO}$(APP)${COLOR_RESET}\ +[${COLOR_VALUE}$(MAKELEVEL)${COLOR_RESET}] \ +${COLOR_HIGHLIGHT}$@${COLOR_RESET}:${COLOR_RESET} ' >&$(WARNING_FD) \ + $(if $(2), \ + && printf '$(1) ' >&$(WARNING_FD) \ + && printf '${COLOR_HIGHLIGHT}$(2)${COLOR_RESET}' >&$(WARNING_FD) \ + $(if $(3),&& printf ' in ${COLOR_VALUE}$(3)${COLOR_RESET}' >&$(WARNING_FD)) \ + , \ + && $(strip $(call PRINTF,$(1)) >&$(WARNING_FD)) \ + ) \ + && printf '\n' >&$(WARNING_FD) + # set ENV=env for targets ending with :env ## for each env in ENV_LIST ## it overrides value of ENV with env diff --git a/make/env.mk b/make/env.mk index 728f0c4..3a846ea 100644 --- a/make/env.mk +++ b/make/env.mk @@ -22,11 +22,11 @@ env = $(env.args) $(env.dist) $(env.file) env.docker = $(env.docker.args) $(env.docker.dist) $(env.docker.file) env.args = $(foreach var,$(ENV_VARS),$(if $($(var)),$(var)='$($(var))')) -env.dist := $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A)' .env.dist - 2>/dev/null) -env.file := $(shell cat $(or $(ENV_FILE),/dev/null) 2>/dev/null |sed '/^[ \t]*$$/d;/^[ \t]*\#/d;') +env.dist ?= $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A)' .env.dist - 2>/dev/null) +env.file ?= $(shell cat $(or $(ENV_FILE),/dev/null) 2>/dev/null |sed '/^[ \t]*$$/d;/^[ \t]*\#/d;s/='\''/=/;s/'\''$$//;s/='\"'/=/;s/'\"'$$//;' |awk -F '=' '{print $$1"='\''"$$2"'\''"}') env.docker.args = $(foreach var,$(ENV_VARS),$(if $($(var)),-e $(var)='$($(var))')) -env.docker.dist := $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A) {print "-e "$$0}' .env.dist - 2>/dev/null) -env.docker.file := $(patsubst %,--env-file %,$(wildcard $(ENV_FILE))) +env.docker.dist ?= $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A) {print "-e "$$0}' .env.dist - 2>/dev/null) +env.docker.file ?= $(patsubst %,--env-file %,$(wildcard $(ENV_FILE))) SHELL:=/bin/bash diff --git a/make/help.mk b/make/help.mk index 33cd5c7..9275d16 100644 --- a/make/help.mk +++ b/make/help.mk @@ -1,13 +1,6 @@ ## # HELP -.DEFAULT_GOAL := help -COLOR_RESET ?= \033[0m -COLOR_GREEN ?= \033[32m -COLOR_BROWN ?= \033[33m -COLOR_BLUE ?= \033[36m -.PHONY: FORCE - # target blank1 blank2: Print new line .PHONY: blank1 blank2 blank1 blank2: @@ -16,7 +9,7 @@ blank1 blank2: # target context: Print Context and Call contexts target .PHONY: context context: - printf "${COLOR_BROWN}Context:${COLOR_RESET}\n" + printf "${COLOR_INFO}Context:${COLOR_RESET}\n" $(MAKE) contexts # target context: Fire context-% target for each CONTEXT @@ -26,10 +19,10 @@ contexts: $(foreach var,$(CONTEXT),context-$(var)) # target context-%: Print % value .PHONY: context-% context-%: - printf "${COLOR_BLUE}%-37s${COLOR_RESET}" $* - printf "${COLOR_GREEN}" + printf "${COLOR_HIGHLIGHT}%-37s${COLOR_RESET}" $* + printf "${COLOR_VALUE}" $(call PRINTF,$($*)) - printf "${COLOR_RESET}" + printf "${COLOR_RESET}\n" # target doc: Fire functions macros target variables doc: functions macros targets variables ; @@ -48,7 +41,7 @@ functions: functions-. # target functions-%: Print documented functions starting with % .PHONY: functions-% functions-%: - awk 'BEGIN {FS = ": "}; $$0 ~ /^# function $*.*:.*$$/ {printf "${COLOR_BLUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) + awk 'BEGIN {FS = ": "}; $$0 ~ /^# function $*.*:.*$$/ {printf "${COLOR_VALUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) # target macros: Fire macros-. .PHONY: macros @@ -57,13 +50,13 @@ macros: macros-. # target macros-%: Print documented macros starting with % .PHONY: macros-% macros-%: - awk 'BEGIN {FS = ": "}; $$0 ~ /^# macro $*.*:.*$$/ {printf "${COLOR_BLUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) + awk 'BEGIN {FS = ": "}; $$0 ~ /^# macro $*.*:.*$$/ {printf "${COLOR_VALUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) # target target: Show common targets .PHONY: target target: - printf "${COLOR_BROWN}Targets:${COLOR_RESET}\n" - awk 'BEGIN {FS = ":.*?## "}; $$0 ~ /^[a-zA-Z_-]+:.*?## .*$$/ {printf "${COLOR_BLUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) + printf "${COLOR_INFO}Targets:${COLOR_RESET}\n" + awk 'BEGIN {FS = ":.*?## "}; $$0 ~ /^[a-zA-Z_-]+:.*?## .*$$/ {printf "${COLOR_VALUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) # target targets: Fire targets-. .PHONY: targets @@ -72,12 +65,12 @@ targets: targets-. # target targets-%: Print documented targets .PHONY: targets-% targets-%: - awk 'BEGIN {FS = ": "}; $$0 ~ /^# target $*.*:.*$$/ {printf "${COLOR_BLUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) + awk 'BEGIN {FS = ": "}; $$0 ~ /^# target $*.*:.*$$/ {printf "${COLOR_VALUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) # target usage: Print Usage .PHONY: usage usage: - printf "${COLOR_BROWN}Usage:${COLOR_RESET}\n" + printf "${COLOR_INFO}Usage:${COLOR_RESET}\n" printf "make [target]\n" # target variables: Fire variables-. @@ -87,4 +80,4 @@ variables: variables-. # target variables-%: Show documented variables .PHONY: variables-% variables-%: - awk 'BEGIN {FS = ": "}; $$0 ~ /^# variable $*.*:.*$$/ {printf "${COLOR_BLUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) + awk 'BEGIN {FS = ": "}; $$0 ~ /^# variable $*.*:.*$$/ {printf "${COLOR_VALUE}%-39s${COLOR_RESET} %s\n", $$1, $$2}' $(MAKEFILE_LIST) diff --git a/make/monorepo/common.mk b/make/monorepo/common.mk index 0d2a24a..b5f1fb1 100644 --- a/make/monorepo/common.mk +++ b/make/monorepo/common.mk @@ -95,7 +95,8 @@ upgrade: upgrade-apps release-upgrade ## Upgrade applications $(APPS): $(if $(wildcard $@/Makefile), \ $(call make,$(patsubst apps-%,%,$(MAKECMDGOALS)) STATUS=0,$(patsubst %/,%,$@),APP_PATH_PREFIX), \ - printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}no app available in folder${COLOR_RESET} $@.\n" >&2) + $(call WARNING,no Makefile in,$@) \ + ) # target apps-%: Fire $(APPS) target to call target % in $(APPS) .PHONY: apps-% diff --git a/make/monorepo/release.mk b/make/monorepo/release.mk index fcc2e70..e118f15 100644 --- a/make/monorepo/release.mk +++ b/make/monorepo/release.mk @@ -46,4 +46,4 @@ release-upgrade: $(patsubst %,release-upgrade-from-%,$(RELEASE_UPGRADE)) release # target release-upgrade-from-%: Sample of catch-all release migration target .PHONY: release-upgrade-from-% release-upgrade-from-%: - echo "Upgrading from release: $*" + printf 'Upgrading from release: $*\n' diff --git a/make/monorepo/subrepo.mk b/make/monorepo/subrepo.mk index bedb6f1..eb6763b 100644 --- a/make/monorepo/subrepo.mk +++ b/make/monorepo/subrepo.mk @@ -3,9 +3,10 @@ # target subrepo-branch-delete: Delete branch $(BRANCH) on remote $(SUBREPO) .PHONY: subrepo-branch-delete -subrepo-branch-delete: myos-base subrepo-check +subrepo-branch-delete: $(if $(DOCKER_RUN),myos-base) subrepo-check ifneq ($(words $(BRANCH)),0) - $(call exec,[ $$(git ls-remote --heads $(REMOTE) $(BRANCH) |wc -l) -eq 1 ] && git push $(REMOTE) :$(BRANCH) || echo Unable to delete branch $(BRANCH) on remote $(REMOTE).) + $(call exec,[ $$(git ls-remote --heads $(REMOTE) $(BRANCH) 2>/dev/null |wc -l) -eq 1 ]) \ + && $(RUN) $(call exec,git push $(REMOTE) :$(BRANCH)) endif # target subrepo-check: Define SUBREPO and REMOTE @@ -27,20 +28,21 @@ endif .PHONY: subrepo-git-diff subrepo-git-diff: myos-base subrepo-check $(eval IGNORE_DRYRUN := true) - $(eval DIFF = $(shell $(call exec,git diff --quiet $(shell $(call exec,git rev-list --ancestry-path $(shell awk '$$1 == "parent" {print $$3}' $(SUBREPO)/.gitrepo)..HEAD |tail -n 1)) -- $(SUBREPO); echo $$?)) ) + $(eval DIFF = $(shell $(call exec,git diff --quiet $(shell $(call exec,git rev-list --ancestry-path $(shell awk '$$1 == "parent" {print $$3}' $(SUBREPO)/.gitrepo)..HEAD |tail -n 1)) -- $(SUBREPO); printf '$$?\n')) ) $(eval IGNORE_DRYRUN := false) # target subrepo-git-fetch: Fetch git remote .PHONY: subrepo-git-fetch subrepo-git-fetch: myos-base subrepo-check - $(call exec,git fetch --prune $(REMOTE)) + $(RUN) $(call exec,git fetch --prune $(REMOTE)) # target subrepo-tag-create-%: Create tag TAG to reference branch REMOTE/% .PHONY: subrepo-tag-create-% subrepo-tag-create-%: myos-base subrepo-check subrepo-git-fetch ifneq ($(words $(TAG)),0) - $(call exec,[ $$(git ls-remote --tags $(REMOTE) $(TAG) |wc -l) -eq 0 ] || git push $(REMOTE) :refs/tags/$(TAG)) - $(call exec,git push $(REMOTE) refs/remotes/subrepo/$(SUBREPO)/$*:refs/tags/$(TAG)) + $(call exec,[ $$(git ls-remote --tags $(REMOTE) $(TAG) |wc -l) -eq 0 ]) \ + || $(RUN) $(call exec,git push $(REMOTE) :refs/tags/$(TAG)) + $(RUN) $(call exec,git push $(REMOTE) refs/remotes/subrepo/$(SUBREPO)/$*:refs/tags/$(TAG)) endif # target subrepo-push: Push to subrepo @@ -50,24 +52,24 @@ subrepo-push: myos-base subrepo-check subrepo-git-fetch subrepo-git-diff ifeq ($(BRANCH),master) $(eval UPDATE_SUBREPO_OPTIONS += -u) endif -# if release|story|hotfix branch, delete remote branch before push and recreate it from master -ifneq ($(findstring $(firstword $(subst /, ,$(BRANCH))),release story hotfix),) +# if specific branch name, delete remote branch before push and recreate it from master +ifneq ($(findstring $(firstword $(subst /, ,$(BRANCH))),feature hotfix release story),) $(eval IGNORE_DRYRUN := true) - $(eval DELETE = $(shell $(call exec,git ls-remote --heads $(REMOTE) $(BRANCH) |wc -l)) ) + $(eval DELETE = $(shell $(call exec,sh -c 'git ls-remote --heads $(REMOTE) $(BRANCH) |wc -l')) ) $(eval IGNORE_DRYRUN := false) else $(eval DELETE = 0) endif if [ $(DIFF) -eq 0 ]; then \ - echo subrepo $(SUBREPO) already up to date.; \ + $(call INFO,subrepo $(SUBREPO) already up to date); \ else \ if [ $(DELETE) -eq 1 ]; then \ - $(call exec,git push $(REMOTE) :$(BRANCH)); \ - $(call exec,git push $(REMOTE) refs/remotes/$(REMOTE)/master:refs/heads/$(BRANCH)); \ + $(RUN) $(call exec,git push $(REMOTE) :$(BRANCH)); \ + $(RUN) $(call exec,git push $(REMOTE) refs/remotes/$(REMOTE)/master:refs/heads/$(BRANCH)); \ fi; \ - $(call exec,git subrepo fetch $(SUBREPO) -b $(BRANCH)); \ - $(call exec,git subrepo push $(SUBREPO) -b $(BRANCH) $(UPDATE_SUBREPO_OPTIONS)); \ - $(call exec,git subrepo clean $(SUBREPO)); \ + $(RUN) $(call exec,git subrepo fetch $(SUBREPO) -b $(BRANCH)); \ + $(RUN) $(call exec,git subrepo push $(SUBREPO) -b $(BRANCH) $(UPDATE_SUBREPO_OPTIONS)); \ + $(RUN) $(call exec,git subrepo clean $(SUBREPO)); \ fi # target subrepos-branch-delete: Fire APPS target @@ -81,7 +83,7 @@ subrepos-tag-create-%: $(APPS) ; # target subrepos-update: Fire APPS target and push updates to upstream .PHONY: subrepos-update subrepos-update: myos-base git-stash $(APPS) git-unstash ## Update subrepos - $(call exec,git push upstream $(BRANCH)) + $(RUN) $(call exec,git push upstream $(BRANCH)) # target subrepo-update-%: Call subrepo-update target in folder % .PHONY: subrepo-update-% diff --git a/make/utils.mk b/make/utils.mk index 854316b..774e566 100644 --- a/make/utils.mk +++ b/make/utils.mk @@ -123,6 +123,7 @@ QUOTE = ' # (that's why the single-quotes are escaped they way they are, above, # and why the dollar sign is not escaped) to printf(1). A trailing # newline is apended, too. +# Note: Removing the apended trailing newline for custom needs # # Note: leading or trailing spaces are *not* stripped. # @@ -130,5 +131,5 @@ define PRINTF printf '$(subst $(sep),\n,\ $(subst $(PERCENT),$(PERCENT)$(PERCENT),\ $(subst $(QUOTE),$(QUOTE)\$(QUOTE)$(QUOTE),\ - $(subst \,\\,$(1)))))\n' + $(subst \,\\,$(1)))))' endef diff --git a/openstack/def.openstack.mk b/openstack/def.openstack.mk index ac450db..7ffb437 100644 --- a/openstack/def.openstack.mk +++ b/openstack/def.openstack.mk @@ -7,16 +7,8 @@ ifneq ($(VERBOSE),) OPENSTACK_ARGS += -v endif -ifeq ($(DOCKER), true) - # function openstack: Call run DOCKER_REPOSITORY/openstack:DOCKER_IMAGE_TAG with arg 1 -define openstack - $(call INFO,openstack,$(1)) - $(call run,$(DOCKER_REPOSITORY)/openstack:$(DOCKER_IMAGE_TAG) $(1)) -endef -else # function openstack: Call run openstack with arg 1 define openstack $(call INFO,openstack,$(1)) - $(call run,openstack $(1)) + $(RUN) $(call run,openstack $(1),$(DOCKER_REPOSITORY)/) endef -endif diff --git a/packer/def.packer.mk b/packer/def.packer.mk index 40c6750..c8feeb6 100644 --- a/packer/def.packer.mk +++ b/packer/def.packer.mk @@ -71,34 +71,19 @@ else ifeq ($(HOST_SYSTEM),WINDOWS) PACKER_QEMU_ACCELERATOR := hax endif -ifeq ($(DOCKER), true) - -# packer ansible provisionner needs: -## empty local ssh agent (ssh-add -D) -## ANSIBLE_SSH_PRIVATE_KEYS set to a key giving access to ANSIBLE_GIT_REPOSITORY without password -## ANSIBLE_AWS_ACCESS_KEY_ID and ANSIBLE_AWS_SECRET_ACCESS_KEY -define packer - $(call run,$(DOCKER_RUN_OPTIONS_PACKER) $(DOCKER_REPOSITORY)/packer:$(DOCKER_IMAGE_TAG) $(1)) -endef -define packer-qemu - echo Running $(1) - $(call run,$(DOCKER_RUN_OPTIONS_PACKER) --entrypoint=qemu-system-$(PACKER_QEMU_ARCH) $(DOCKER_REPOSITORY)/packer:$(DOCKER_IMAGE_TAG) $(PACKER_QEMU_ARGS) -m 512m -drive file=$(1)$(comma)format=raw -net nic$(comma)model=virtio -net user$(comma)hostfwd=tcp:$(PACKER_SSH_ADDRESS):$(PACKER_SSH_PORT)-:22 -vnc $(PACKER_VNC_ADDRESS):$(subst 590,,$(PACKER_VNC_PORT))) -endef - -else - # function packer: Call run packer with arg 1 +## it needs an empty local ssh agent (ssh-add -D) +## it needs ANSIBLE_SSH_PRIVATE_KEYS set to a key giving access to ANSIBLE_GIT_REPOSITORY without password +## it needs ANSIBLE_AWS_ACCESS_KEY_ID and ANSIBLE_AWS_SECRET_ACCESS_KEY define packer - $(call run,packer $(1)) + $(RUN) $(call run,packer $(1),$(DOCKER_RUN_OPTIONS_PACKER) $(DOCKER_REPOSITORY)/) endef # function packer-qemu: Call run qemu-system-% for PACKER_QEMU_ARCH define packer-qemu echo Running $(1) - $(call run,qemu-system-$(PACKER_QEMU_ARCH) $(PACKER_QEMU_ARGS) -m 512m -drive file=$(1)$(comma)format=raw -net nic$(comma)model=virtio -net user$(comma)hostfwd=tcp:$(PACKER_SSH_ADDRESS):$(PACKER_SSH_PORT)-:22 -vnc $(PACKER_VNC_ADDRESS):$(subst 590,,$(PACKER_VNC_PORT))) + $(RUN) $(call run,$(if $(DOCKER_RUN),packer,qemu-system-$(PACKER_QEMU_ARCH)) $(PACKER_QEMU_ARGS) -m 512m -drive file=$(1)$(comma)format=raw -net nic$(comma)model=virtio -net user$(comma)hostfwd=tcp:$(PACKER_SSH_ADDRESS):$(PACKER_SSH_PORT)-:22 -vnc $(PACKER_VNC_ADDRESS):$(subst 590,,$(PACKER_VNC_PORT)),$(DOCKER_RUN_OPTIONS_PACKER) --entrypoint=qemu-system-$(PACKER_QEMU_ARCH) $(DOCKER_REPOSITORY)/) endef -endif - # function packer-build: Call packer build with arg 1, Add build infos to file PACKER_ISO_INFO define packer-build $(eval PACKER_TEMPLATE := $(notdir $(basename $(1)))) @@ -125,6 +110,4 @@ define packer-build echo 'user: $(username)' >> $(PACKER_ISO_INFO) endef -define arrays_of_dquoted_args -[ $(subst $(dquote) $(dquote),$(dquote)$(comma) $(dquote),$(subst $(dquote) $(dquote)-,$(dquote) ]$(comma) [ $(dquote)-,$(patsubst %,$(dquote)%$(dquote),$1))) ] -endef +arrays_of_dquoted_args = [ $(subst $(dquote) $(dquote),$(dquote)$(comma) $(dquote),$(subst $(dquote) $(dquote)-,$(dquote) ]$(comma) [ $(dquote)-,$(patsubst %,$(dquote)%$(dquote),$1))) ] diff --git a/stack/base.mk b/stack/base.mk index affe27b..c3f855c 100644 --- a/stack/base.mk +++ b/stack/base.mk @@ -1,25 +1,3 @@ # target base: Fire docker-network-create stack-base-up base-ssh-add .PHONY: base -base: docker-network-create $(if $(filter $(DOCKER),true),stack-base-up base-ssh-add) - -# target ssh-add: Fire base-ssh-add -.PHONY: ssh-add -ssh-add: base-ssh-add - -# target base-ssh-add: Fire base-ssh-key and exec ssh-add file SSH_PRIVATE_KEYS in folder SSH_DIR -.PHONY: base-ssh-add -base-ssh-add: base-ssh-key - $(eval SSH_PRIVATE_KEYS := $(foreach file,$(SSH_DIR)/id_rsa $(filter-out $(wildcard $(SSH_DIR)/id_rsa),$(wildcard $(SSH_DIR)/*)),$(if $(shell grep "PRIVATE KEY" $(file) 2>/dev/null),$(notdir $(file))))) - $(eval IGNORE_VERBOSE := true) - $(call docker-run,$(DOCKER_IMAGE_CLI),sh -c "$(foreach file,$(patsubst %,$(SSH_DIR)/%,$(SSH_PRIVATE_KEYS)),ssh-add -l |grep -qw $$(ssh-keygen -lf $(file) 2>/dev/null |awk '{print $$2}') 2>/dev/null || ssh-add $(file) ||: &&) true") - $(eval IGNORE_VERBOSE := false) - -# target base-ssh-key: Setup ssh private key SSH_KEY in SSH_DIR -.PHONY: base-ssh-key -base-ssh-key: stack-base-up -ifneq (,$(filter true,$(DRONE))) - $(call exec,[ ! -d $(SSH_DIR) ] && mkdir -p $(SSH_DIR) && chown $(UID) $(SSH_DIR) && chmod 0700 $(SSH_DIR) ||:) -else - $(eval DOCKER_RUN_VOLUME += -v $(SSH_DIR):$(SSH_DIR)) -endif - $(if $(SSH_KEY),$(eval export SSH_KEY ?= $(SSH_KEY)) $(call docker-run,$(DOCKER_IMAGE_CLI),echo -e "$$SSH_KEY" > $(SSH_DIR)/${COMPOSE_PROJECT_NAME}_id_rsa && chmod 0400 $(SSH_DIR)/${COMPOSE_PROJECT_NAME}_id_rsa && chown $(UID) $(SSH_DIR)/${COMPOSE_PROJECT_NAME}_id_rsa ||:)) +base: docker-network-create $(if $(DOCKER_RUN),stack-base-up) ssh-add diff --git a/stack/base/base.yml b/stack/base/base.yml index d044aab..3c6925b 100644 --- a/stack/base/base.yml +++ b/stack/base/base.yml @@ -5,14 +5,6 @@ services: build: args: - DOCKER_BUILD_DIR=docker/cli - - GIT_AUTHOR_NAME=${GIT_AUTHOR_NAME} - - GIT_AUTHOR_EMAIL=${GIT_AUTHOR_EMAIL} - - SSH_BASTION_USERNAME=${SSH_BASTION_USERNAME} - - SSH_BASTION_HOSTNAME=${SSH_BASTION_HOSTNAME} - - SSH_PUBLIC_HOST_KEYS=${SSH_PUBLIC_HOST_KEYS} - - SSH_PRIVATE_IP_RANGE=${SSH_PRIVATE_IP_RANGE} - - UID=${UID} - - USER=${USER} context: ../.. dockerfile: docker/cli/Dockerfile target: ${DOCKER_BUILD_TARGET} @@ -34,8 +26,6 @@ services: build: args: - DOCKER_BUILD_DIR=docker/ssh - - UID=${UID} - - USER=${USER} context: ../.. dockerfile: docker/ssh/Dockerfile target: ${DOCKER_BUILD_TARGET} @@ -49,11 +39,6 @@ services: volumes: ssh: - driver: local - driver_opts: - type: tmpfs - device: tmpfs - o: uid=${UID} networks: private: diff --git a/stack/node.mk b/stack/node.mk index e3127bf..ccf47f5 100644 --- a/stack/node.mk +++ b/stack/node.mk @@ -1,13 +1,5 @@ ENV_VARS += DOCKER_HOST_IFACE DOCKER_HOST_INET -# target node: Fire docker-network-create-% for DOCKER_NETWORK_PUBLIC node-openssl stack-node-up +# target node: Fire docker-network-create-% for DOCKER_NETWORK_PUBLIC ssl-certs stack-node-up .PHONY: node -node: docker-network-create-$(DOCKER_NETWORK_PUBLIC) node-openssl stack-node-up - -# target node-openssl: Create /certs/${DOMAIN}.key.pem and /certs/${DOMAIN}.crt.pem files -.PHONY: node-openssl -node-openssl: - docker run --rm --mount source=$(COMPOSE_PROJECT_NAME_NODE)_ssl-certs,target=/certs alpine:latest [ -f /certs/$(DOMAIN).crt.pem -a -f /certs/$(DOMAIN).key.pem ] \ - || docker run --rm -e DOMAIN=$(DOMAIN) --mount source=$(COMPOSE_PROJECT_NAME_NODE)_ssl-certs,target=/certs alpine:latest sh -c "apk --no-cache add openssl \ - && { [ -f /certs/${DOMAIN}.key.pem ] || openssl genrsa -out /certs/${DOMAIN}.key.pem 2048; } \ - && openssl req -key /certs/${DOMAIN}.key.pem -out /certs/${DOMAIN}.crt.pem -addext extendedKeyUsage=serverAuth -addext subjectAltName=DNS:${DOMAIN} -subj \"/C=/ST=/L=/O=/CN=${DOMAIN}\" -x509 -days 365" +node: docker-network-create-$(DOCKER_NETWORK_PUBLIC) ssl-certs stack-node-up diff --git a/stack/sematext/logagent.yml b/stack/sematext/logagent.yml index 550aa87..a67ee94 100644 --- a/stack/sematext/logagent.yml +++ b/stack/sematext/logagent.yml @@ -5,10 +5,6 @@ services: build: args: - DOCKER_BUILD_DIR=docker/sematext/logagent - - GIT_AUTHOR_NAME=${GIT_AUTHOR_NAME} - - GIT_AUTHOR_EMAIL=${GIT_AUTHOR_EMAIL} - - UID=${UID} - - USER=${USER} context: ../.. dockerfile: docker/sematext/logagent/Dockerfile target: ${DOCKER_BUILD_TARGET} diff --git a/stack/theia/theia.yml b/stack/theia/theia.yml index f5b7ae0..50ea19e 100644 --- a/stack/theia/theia.yml +++ b/stack/theia/theia.yml @@ -5,15 +5,6 @@ services: build: args: - DOCKER_BUILD_DIR=docker/theia - - DOCKER_GID=${DOCKER_GID} - - GIT_AUTHOR_NAME=${GIT_AUTHOR_NAME} - - GIT_AUTHOR_EMAIL=${GIT_AUTHOR_EMAIL} - - SSH_BASTION_USERNAME=${SSH_BASTION_USERNAME} - - SSH_BASTION_HOSTNAME=${SSH_BASTION_HOSTNAME} - - SSH_PUBLIC_HOST_KEYS=${SSH_PUBLIC_HOST_KEYS} - - SSH_PRIVATE_IP_RANGE=${SSH_PRIVATE_IP_RANGE} - - UID=${UID} - - USER=${USER} context: ../.. dockerfile: docker/theia/Dockerfile target: ${DOCKER_BUILD_TARGET} diff --git a/terraform/def.terraform.mk b/terraform/def.terraform.mk index 5648cd0..64fda60 100644 --- a/terraform/def.terraform.mk +++ b/terraform/def.terraform.mk @@ -1,17 +1,5 @@ CMDS += terraform -ifeq ($(DOCKER), true) - define terraform - $(call run,hashicorp/terraform:light $(1)) + $(RUN) $(call run,terraform $(1),hashicorp/) endef - -else - -define terraform - $(call run,terraform $(1)) -endef - -endif - -