diff --git a/ansible/def.ansible.mk b/ansible/def.ansible.mk index 0be9c0c..db3b273 100644 --- a/ansible/def.ansible.mk +++ b/ansible/def.ansible.mk @@ -20,6 +20,7 @@ ANSIBLE_SERVER_NAME ?= $(SERVER_NAME) ANSIBLE_USERNAME ?= root ANSIBLE_VERBOSE ?= -v CMDS += ansible ansible-playbook +DOCKER_RUN_OPTIONS += --add-host=host.docker.internal:$(DOCKER_INTERNAL_DOCKER_HOST) ENV_VARS += ANSIBLE_AWS_ACCESS_KEY_ID ANSIBLE_AWS_DEFAULT_OUTPUT ANSIBLE_AWS_DEFAULT_REGION ANSIBLE_AWS_SECRET_ACCESS_KEY ANSIBLE_CONFIG ANSIBLE_DISKS_NFS_DISK ANSIBLE_DISKS_NFS_OPTIONS ANSIBLE_DISKS_NFS_PATH ANSIBLE_DOCKER_IMAGE_TAG ANSIBLE_DOCKER_REGISTRY ANSIBLE_EXTRA_VARS ANSIBLE_GIT_DIRECTORY ANSIBLE_GIT_KEY_FILE ANSIBLE_GIT_REPOSITORY ANSIBLE_GIT_VERSION ANSIBLE_INVENTORY ANSIBLE_PLAYBOOK ANSIBLE_SSH_PRIVATE_KEYS ANSIBLE_USERNAME ANSIBLE_VERBOSE ifeq ($(DEBUG), true) @@ -28,14 +29,14 @@ endif ifeq ($(DOCKER), true) define ansible - $(call run,$(DOCKER_SSH_AUTH) -v ~/.aws:/home/$(USER)/.aws --add-host=host.docker.internal:$(DOCKER_INTERNAL_DOCKER_HOST) $(DOCKER_REPOSITORY)/ansible:$(DOCKER_IMAGE_TAG) $(ANSIBLE_ARGS) -i $(ANSIBLE_INVENTORY)/.host.docker.internal $(ANSIBLE_VERBOSE) $(1)) + $(call run,$(DOCKER_REPOSITORY)/ansible:$(DOCKER_IMAGE_TAG) $(ANSIBLE_ARGS) -i $(ANSIBLE_INVENTORY)/.host.docker.internal $(ANSIBLE_VERBOSE) $(1)) endef define ansible-playbook - $(call run,$(DOCKER_SSH_AUTH) -v ~/.aws:/home/$(USER)/.aws --add-host=host.docker.internal:$(DOCKER_INTERNAL_DOCKER_HOST) --entrypoint=ansible-playbook $(DOCKER_REPOSITORY)/ansible:$(DOCKER_IMAGE_TAG) $(ANSIBLE_ARGS) -i $(ANSIBLE_INVENTORY)/.host.docker.internal $(ANSIBLE_VERBOSE) $(1)) + $(call run,--entrypoint=ansible-playbook $(DOCKER_REPOSITORY)/ansible:$(DOCKER_IMAGE_TAG) $(ANSIBLE_ARGS) -i $(ANSIBLE_INVENTORY)/.host.docker.internal $(ANSIBLE_VERBOSE) $(1)) endef define ansible-pull # TODO : run ansible in docker and target localhost outside docker - IFS=$$'\n'; $(ECHO) env $(foreach var,$(ENV_VARS),$(if $($(var)),$(var)='$($(var))')) $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A)' .env.dist - 2>/dev/null) $$(cat $(ENV_FILE) 2>/dev/null |awk -F "=" '$$1 ~! /^\(#|$$\)/') ansible-pull $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1) + $(call env-exec,ansible-pull $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) endef else # function ansible: Call run ansible ANSIBLE_ARGS with arg 1 diff --git a/aws/def.aws.mk b/aws/def.aws.mk index 9811550..2758e5e 100644 --- a/aws/def.aws.mk +++ b/aws/def.aws.mk @@ -10,11 +10,12 @@ AWS_S3_KEY ?= $(PACKER_ISO_FILE) AWS_SECRET_ACCESS_KEY ?= $(shell $(call conf,$(HOME)/.aws/credentials,$(or $(AWS_PROFILE),default),aws_secret_access_key)) AWS_SNAP_DESCRIPTION ?= iso: $(AWS_S3_KEY) env: $(ENV) app: $(APP) branch: $(BRANCH) version: $(VERSION) user: $(USER) etag: $(AWS_S3_KEY_ETAG) date: $(AWS_S3_KEY_DATE) CMDS += aws +DOCKER_RUN_VOLUME += -v $(HOME)/.aws:/home/$(USER)/.aws ENV_VARS += AWS_ACCESS_KEY_ID AWS_AMI_DESCRIPTION AWS_AMI_NAME AWS_DEFAULT_OUTPUT AWS_DEFAULT_REGION AWS_INSTANCE_ID AWS_PROFILE AWS_S3_BUCKET AWS_S3_KEY AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_SNAP_DESCRIPTION AWS_SNAP_ID ifeq ($(DOCKER), true) define aws - $(call run,$(DOCKER_SSH_AUTH) -v $$HOME/.aws:/root/.aws:ro anigeo/awscli:latest $(1)) + $(call run,$(DOCKER_REPOSITORY)/aws:$(DOCKER_IMAGE_TAG) $(1)) endef else # function aws: Call run aws with arg 1 diff --git a/make/common.mk b/make/common.mk index 70b8236..3768495 100644 --- a/make/common.mk +++ b/make/common.mk @@ -7,13 +7,6 @@ $(APP): APP_DIR := $(RELATIVE)$(APP) $(APP): myos-base $(call update-app) -# target $(CONFIG): Update config files -.PHONY: $(CONFIG) -$(CONFIG): SSH_PUBLIC_HOST_KEYS := $(CONFIG_REMOTE_HOST) $(SSH_BASTION_HOSTNAME) $(SSH_REMOTE_HOSTS) -$(CONFIG): MAKE_VARS += SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PRIVATE_IP_RANGE SSH_PUBLIC_HOST_KEYS -$(CONFIG): myos-base - $(call update-app,$(CONFIG_REPOSITORY),$(CONFIG)) - # target install-app install-apps: Call install-app for each ARGS .PHONY: install-app install-apps install-app install-apps: myos-base install-app-required @@ -41,9 +34,12 @@ update-app: update-app-$(APP_NAME) ; .PHONY: update-app-% update-app-%: % ; -# target update-config: Fire CONFIG +# target update-config: Update config files .PHONY: update-config -update-config: $(CONFIG) +update-config: SSH_PUBLIC_HOST_KEYS := $(CONFIG_REMOTE_HOST) $(SSH_BASTION_HOSTNAME) $(SSH_REMOTE_HOSTS) +update-config: MAKE_VARS += SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PRIVATE_IP_RANGE SSH_PUBLIC_HOST_KEYS +update-config: myos-base + $(call update-app,$(CONFIG_REPOSITORY),$(CONFIG)) # target update-hosts: Update /etc/hosts # on local host diff --git a/make/def.docker.mk b/make/def.docker.mk index b6c316b..b265513 100644 --- a/make/def.docker.mk +++ b/make/def.docker.mk @@ -32,6 +32,15 @@ else DOCKER_RUN_VOLUME := -v /var/run/docker.sock:/var/run/docker.sock -v $(or $(MONOREPO_DIR),$(APP_DIR)):$(or $(WORKSPACE_DIR),$(MONOREPO_DIR),$(APP_DIR)) endif +# function env-run: Call env-exec with arg 1 in a subshell +define env-run + $(call env-exec,sh -c '$(or $(1),$(SHELL))') +endef +# function env-exec: Exec arg 1 in a new env +define env-exec + IFS=$$'\n'; env $(env_reset) $(env) $(1) +endef + ifeq ($(DOCKER), true) DOCKER_SSH_AUTH := -e SSH_AUTH_SOCK=/tmp/ssh-agent/socket -v $(DOCKER_VOLUME_SSH):/tmp/ssh-agent @@ -43,17 +52,17 @@ endef ifeq ($(DRONE), true) # function exec: Run new DOCKER_IMAGE docker with arg 1 define exec - $(call run,$(DOCKER_SSH_AUTH) $(DOCKER_IMAGE) sh -c '$(or $(1),$(SHELL))') + $(call run,$(DOCKER_IMAGE) sh -c '$(or $(1),$(SHELL))') endef else # function exec: Exec arg 1 in docker DOCKER_NAME define exec - $(ECHO) docker exec $(DOCKER_EXEC_OPTIONS) $(DOCKER_ENV) $(DOCKER_RUN_WORKDIR) $(DOCKER_NAME) sh -c '$(or $(1),$(SHELL))' + $(ECHO) docker exec $(DOCKER_ENV) $(DOCKER_EXEC_OPTIONS) $(DOCKER_RUN_WORKDIR) $(DOCKER_NAME) sh -c '$(or $(1),$(SHELL))' endef endif # function run: Pass arg 1 to docker run define run - $(ECHO) docker run $(DOCKER_RUN_OPTIONS) $(DOCKER_ENV) $(DOCKER_RUN_VOLUME) $(DOCKER_RUN_WORKDIR) $(1) + $(ECHO) docker run $(DOCKER_ENV) $(DOCKER_RUN_OPTIONS) $(DOCKER_RUN_VOLUME) $(DOCKER_RUN_WORKDIR) $(DOCKER_SSH_AUTH) $(1) endef else @@ -63,13 +72,13 @@ SHELL := /bin/bash define docker-run $(ECHO) docker run $(DOCKER_RUN_OPTIONS) $(DOCKER_ENV) $(DOCKER_RUN_VOLUME) $(DOCKER_RUN_WORKDIR) $(or $(1),$(DOCKER_IMAGE):$(DOCKER_IMAGE_TAG)) $(2) endef -# function exec: Call run with arg 1 +# function exec: Call env-exec with arg 1 or SHELL define exec - $(call run,sh -c '$(or $(1),$(SHELL))') + $(call env-exec,$(or $(1),$(SHELL))) endef -# function run: Exec arg 1 +# function run: Call env-run with arg 1 define run - IFS=$$'\n'; env $(env_reset) $(env) $(1) + $(call env-run,$(1)) endef endif diff --git a/make/def.mk b/make/def.mk index 8334c55..a33d11b 100644 --- a/make/def.mk +++ b/make/def.mk @@ -13,7 +13,7 @@ COMMIT ?= $(or $(SUBREPO_COMMIT),$(GIT_COMMIT)) CONFIG ?= $(RELATIVE)config CONFIG_REPOSITORY ?= $(call pop,$(or $(APP_UPSTREAM_REPOSITORY),$(GIT_UPSTREAM_REPOSITORY)))/$(notdir $(CONFIG)) CONTEXT ?= $(if $(APP),APP BRANCH VERSION) $(shell awk 'BEGIN {FS="="}; $$1 !~ /^(\#|$$)/ {print $$1}' .env.dist 2>/dev/null) -CONTEXT_DEBUG ?= MAKEFILE_LIST env APPS GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME LOG_LEVEL MAKE_DIR MAKE_SUBDIRS MAKE_CMD_ARGS MAKE_ENV_ARGS MONOREPO_DIR UID USER +CONTEXT_DEBUG ?= MAKEFILE_LIST env env.docker APPS GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME LOG_LEVEL MAKE_DIR MAKE_SUBDIRS MAKE_CMD_ARGS MAKE_ENV_ARGS MONOREPO_DIR UID USER DEBUG ?= false DOCKER ?= true DOMAIN ?= localhost @@ -22,7 +22,7 @@ DRYRUN ?= false DRYRUN_IGNORE ?= false DRYRUN_RECURSIVE ?= false ENV ?= dist -ENV_FILE ?= $(wildcard $(CONFIG)/$(ENV)/$(APP)/.env) .env +ENV_FILE ?= $(wildcard $(CONFIG)/$(ENV)/$(APP)/.env .env) ENV_LIST ?= debug local tests release master #TODO: staging develop ENV_RESET ?= false ENV_VARS ?= APP BRANCH ENV HOSTNAME GID GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME MONOREPO MONOREPO_DIR TAG UID USER VERSION @@ -180,8 +180,8 @@ sed = $(call exec,sed -i $(SED_SUFFIX) '\''$(1)'\'' $(2)) define install-app $(eval url := $(or $(1), $(APP_REPOSITORY))) $(eval dir := $(or $(2), $(RELATIVE)$(lastword $(subst /, ,$(url))))) - [ -d $(dir) ] && $(call update-app,$(url),$(dir)) - [ -d $(dir) ] || $(call exec,$(ECHO) git clone $(QUIET) $(url) $(dir)) + [ -d $(dir)/.git ] && $(call update-app,$(url),$(dir)) + [ -d $(dir)/.git ] || $(call exec,$(ECHO) git clone $(QUIET) $(url) $(dir)) endef # function update-app: Exec 'cd dir 1 && git pull' or Call install-app @@ -189,8 +189,8 @@ endef define update-app $(eval url := $(or $(1), $(APP_REPOSITORY))) $(eval dir := $(or $(2), $(APP_DIR))) - [ -d $(dir) ] && $(call exec,cd $(dir) && $(ECHO) git pull $(QUIET)) - [ -d $(dir) ] || $(call install-app,$(url),$(dir)) + [ -d $(dir)/.git ] && $(call exec,cd $(dir) && $(ECHO) git pull $(QUIET)) + [ -d $(dir)/.git ] || $(call install-app,$(url),$(dir)) endef # function TARGET:ENV: Create a new target ending with :env diff --git a/make/env.mk b/make/env.mk index 3d60b0d..47e5cc6 100644 --- a/make/env.mk +++ b/make/env.mk @@ -22,11 +22,11 @@ env = $(env.args) $(env.dist) $(env.file) env.docker = $(env.docker.args) $(env.docker.dist) $(env.docker.file) env.args = $(foreach var,$(ENV_VARS),$(if $($(var)),$(var)='$($(var))')) -env.dist = $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A)' .env.dist - 2>/dev/null) -env.file = $(shell cat $(ENV_FILE) 2>/dev/null |sed '/^[ \t]*$$/d;/^[ \t]*\#/d;') +env.dist := $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A)' .env.dist - 2>/dev/null) +env.file := $(shell cat $(or $(ENV_FILE),/dev/null) 2>/dev/null |sed '/^[ \t]*$$/d;/^[ \t]*\#/d;') env.docker.args = $(foreach var,$(ENV_VARS),$(if $($(var)),-e $(var)='$($(var))')) -env.docker.dist = $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A) {print "-e "$$0}' .env.dist - 2>/dev/null) -env.docker.file = $(patsubst %,--env-file %,$(wildcard $(ENV_FILE))) +env.docker.dist := $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A) {print "-e "$$0}' .env.dist - 2>/dev/null) +env.docker.file := $(patsubst %,--env-file %,$(wildcard $(ENV_FILE))) SHELL:=/bin/bash diff --git a/openstack/def.openstack.mk b/openstack/def.openstack.mk index 2e91493..aee5d76 100644 --- a/openstack/def.openstack.mk +++ b/openstack/def.openstack.mk @@ -9,7 +9,7 @@ endif ifeq ($(DOCKER), true) define openstack - $(call run,$(DOCKER_SSH_AUTH) $(DOCKER_REPOSITORY)/openstack:$(DOCKER_IMAGE_TAG) $(1)) + $(call run,$(DOCKER_REPOSITORY)/openstack:$(DOCKER_IMAGE_TAG) $(1)) endef else # function openstack: Call run openstack with arg 1 diff --git a/packer/def.packer.mk b/packer/def.packer.mk index 8a4d38b..bcc9671 100644 --- a/packer/def.packer.mk +++ b/packer/def.packer.mk @@ -1,4 +1,5 @@ CMDS += packer +DOCKER_RUN_OPTIONS_PACKER ?= -p $(PACKER_SSH_PORT):$(PACKER_SSH_PORT) -p $(PACKER_VNC_PORT):$(PACKER_VNC_PORT) ENV_VARS += PACKER_CACHE_DIR PACKER_KEY_INTERVAL PACKER_LOG KVM_GID ?= $(call gid,kvm) PACKER_ARCH ?= $(PACKER_ALPINE_ARCH) @@ -65,7 +66,7 @@ PACKER_QEMU_ACCELERATOR := tcg PACKER_QEMU_ARGS += -cpu max,vendor=GenuineIntel,vmware-cpuid-freq=on,+invtsc,+aes,+vmx endif else ifeq ($(HOST_SYSTEM),LINUX) -DOCKER_RUN_OPTIONS_PACKER := $(if $(KVM_GID),--group-add $(KVM_GID)) --device /dev/kvm +DOCKER_RUN_OPTIONS_PACKER += $(if $(KVM_GID),--group-add $(KVM_GID)) --device /dev/kvm else ifeq ($(HOST_SYSTEM),WINDOWS) PACKER_QEMU_ACCELERATOR := hax endif @@ -77,11 +78,11 @@ ifeq ($(DOCKER), true) ## ANSIBLE_SSH_PRIVATE_KEYS set to a key giving access to ANSIBLE_GIT_REPOSITORY without password ## ANSIBLE_AWS_ACCESS_KEY_ID and ANSIBLE_AWS_SECRET_ACCESS_KEY define packer - $(call run,$(DOCKER_RUN_OPTIONS_PACKER) $(DOCKER_SSH_AUTH) -p $(PACKER_SSH_PORT):$(PACKER_SSH_PORT) -p $(PACKER_VNC_PORT):$(PACKER_VNC_PORT) $(DOCKER_REPOSITORY)/packer:$(DOCKER_IMAGE_TAG) $(1)) + $(call run,$(DOCKER_RUN_OPTIONS_PACKER) $(DOCKER_REPOSITORY)/packer:$(DOCKER_IMAGE_TAG) $(1)) endef define packer-qemu echo Running $(1) - $(call run,$(DOCKER_RUN_OPTIONS_PACKER) -p $(PACKER_SSH_PORT):$(PACKER_SSH_PORT) -p $(PACKER_VNC_PORT):$(PACKER_VNC_PORT) --entrypoint=qemu-system-$(PACKER_QEMU_ARCH) $(DOCKER_REPOSITORY)/packer:$(DOCKER_IMAGE_TAG) $(PACKER_QEMU_ARGS) -m 512m -drive file=$(1)$(comma)format=raw -net nic$(comma)model=virtio -net user$(comma)hostfwd=tcp:$(PACKER_SSH_ADDRESS):$(PACKER_SSH_PORT)-:22 -vnc $(PACKER_VNC_ADDRESS):$(subst 590,,$(PACKER_VNC_PORT))) + $(call run,$(DOCKER_RUN_OPTIONS_PACKER) --entrypoint=qemu-system-$(PACKER_QEMU_ARCH) $(DOCKER_REPOSITORY)/packer:$(DOCKER_IMAGE_TAG) $(PACKER_QEMU_ARGS) -m 512m -drive file=$(1)$(comma)format=raw -net nic$(comma)model=virtio -net user$(comma)hostfwd=tcp:$(PACKER_SSH_ADDRESS):$(PACKER_SSH_PORT)-:22 -vnc $(PACKER_VNC_ADDRESS):$(subst 590,,$(PACKER_VNC_PORT))) endef else diff --git a/stack/base.mk b/stack/base.mk index afaa03c..5f22cd9 100644 --- a/stack/base.mk +++ b/stack/base.mk @@ -10,7 +10,7 @@ ssh-add: base-ssh-add .PHONY: base-ssh-add base-ssh-add: base-ssh-key $(eval SSH_PRIVATE_KEYS := $(foreach file,$(SSH_DIR)/id_rsa $(filter-out $(wildcard $(SSH_DIR)/id_rsa),$(wildcard $(SSH_DIR)/*)),$(if $(shell grep "PRIVATE KEY" $(file) 2>/dev/null),$(notdir $(file))))) - $(call docker-run,$(DOCKER_SSH_AUTH) $(DOCKER_IMAGE_CLI),sh -c "$(foreach file,$(patsubst %,$(SSH_DIR)/%,$(SSH_PRIVATE_KEYS)),ssh-add -l |grep -qw $$(ssh-keygen -lf $(file) 2>/dev/null |awk '{print $$2}') 2>/dev/null || ssh-add $(file) ||: &&) true") + $(call docker-run,$(DOCKER_IMAGE_CLI),sh -c "$(foreach file,$(patsubst %,$(SSH_DIR)/%,$(SSH_PRIVATE_KEYS)),ssh-add -l |grep -qw $$(ssh-keygen -lf $(file) 2>/dev/null |awk '{print $$2}') 2>/dev/null || ssh-add $(file) ||: &&) true") # target base-ssh-key: Setup ssh private key SSH_KEY in SSH_DIR .PHONY: base-ssh-key