From 7271c1160f87a22aa9b459222c6136c0c83b0d5b Mon Sep 17 00:00:00 2001 From: "aynic.os" Date: Mon, 17 May 2021 03:40:02 +0200 Subject: [PATCH] wip --- .env.dist | 1 - make/apps/build.mk | 2 +- make/apps/common.mk | 24 ++++++++--------- make/apps/def.docker.mk | 4 +-- make/apps/deploy.mk | 67 ++-------------------------------------------- make/apps/docker.mk | 41 +++++++++++++++++++++------- make/apps/infra/clean.mk | 18 ------------- make/apps/install.mk | 7 ++--- make/def.docker.mk | 20 +++++++------- make/def.mk | 54 +++++++++++++++++++------------------ make/include.mk | 2 +- make/infra.mk | 10 +++---- make/monorepo/common.mk | 2 +- make/monorepo/def.drone.mk | 4 +-- make/monorepo/def.mk | 2 +- make/monorepo/git.mk | 20 +++++++------- make/monorepo/install.mk | 4 +-- make/monorepo/subrepo.mk | 6 ++--- make/monorepo/update.mk | 12 ++++----- stack/elastic.mk | 5 ++++ 20 files changed, 125 insertions(+), 180 deletions(-) delete mode 100644 make/apps/infra/clean.mk diff --git a/.env.dist b/.env.dist index a7a04e1..591f177 100644 --- a/.env.dist +++ b/.env.dist @@ -1,4 +1,3 @@ -APP=yaip APP_DOMAIN=${ENV}.${DOMAIN} APP_HOST=${APP}.${APP_DOMAIN} APP_NAME=${APP} diff --git a/make/apps/build.mk b/make/apps/build.mk index 01021ce..a680e43 100644 --- a/make/apps/build.mk +++ b/make/apps/build.mk @@ -7,7 +7,7 @@ build: docker-compose-build ## Build application docker images # target build@%: Build application docker images to deploy .PHONY: build@% app-build -build@%: infra-base +build@%: myos-base $(eval DRYRUN_IGNORE := true) $(eval SERVICES ?= $(shell $(call docker-compose,--log-level critical config --services))) $(eval DRYRUN_IGNORE := false) diff --git a/make/apps/common.mk b/make/apps/common.mk index e85ff24..c997bfc 100644 --- a/make/apps/common.mk +++ b/make/apps/common.mk @@ -2,20 +2,20 @@ # COMMON .PHONY: bootstrap -bootstrap: bootstrap-git bootstrap-docker ## Bootstrap application +bootstrap: bootstrap-git bootstrap-docker app-bootstrap ## Bootstrap application .PHONY: boostrap-docker bootstrap-docker: docker-network-create - $(if $(filter bootstrap-$(APP),$(MAKETARGETS)),$(call make,bootstrap-$(APP))) $(call make,docker-compose-up) .PHONY: bootstrap-git -bootstrap-git: -ifneq ($(SUBREPO),) - if ! git config remote.subrepo/$(SUBREPO).url > /dev/null ; \ - then git remote add subrepo/$(SUBREPO) $(GIT_REPOSITORY); \ +bootstrap-git: bootstrap-git-$(APP_DIR) + +.PHONY: bootstrap-git-% +bootstrap-git-%: + if ! git config remote.origin.url > /dev/null ; \ + then git clone $(GIT_REPOSITORY) $*; \ fi -endif .PHONY: config config: docker-compose-config ## View docker compose file @@ -26,7 +26,7 @@ connect: docker-compose-connect ## Connect to docker $(SERVICE) .PHONY: connect@% connect@%: SERVICE ?= $(DOCKER_SERVICE) connect@%: ## Connect to docker $(SERVICE) on first remote host - $(call make,ssh-connect,../infra,APP SERVICE) + $(call make,ssh-connect,$(MYOS),APP SERVICE) .PHONY: down down: docker-compose-down ## Remove application dockers @@ -42,7 +42,7 @@ endif .PHONY: exec@% exec@%: SERVICE ?= $(DOCKER_SERVICE) exec@%: ## Exec a command in docker $(SERVICE) on all remote hosts - $(call make,ssh-exec,../infra,APP ARGS SERVICE) + $(call make,ssh-exec,$(MYOS),APP ARGS SERVICE) .PHONY: logs logs: docker-compose-logs ## Display application dockers logs @@ -72,14 +72,14 @@ endif .PHONY: run@% run@%: SERVICE ?= $(DOCKER_SERVICE) run@%: ## Run a command on all remote hosts - $(call make,ssh-run,../infra,APP ARGS) + $(call make,ssh-run,$(MYOS),APP ARGS) .PHONY: scale scale: docker-compose-scale ## Scale application to NUM dockers .PHONY: ssh@% ssh@%: ## Connect to first remote host - $(call make,ssh,../infra,APP) + $(call make,ssh,$(MYOS),APP) # target stack: Call docker-stack function with each value of $(STACK) .PHONY: stack @@ -96,7 +96,7 @@ stack-%: $(eval command := $(lastword $(subst -, ,$*))) $(if $(findstring -,$*), \ $(if $(filter $(command),$(filter-out %-%,$(patsubst docker-compose-%,%,$(filter docker-compose-%,$(MAKETARGETS))))), \ - $(call make,docker-compose-$(command) STACK="$(stack)" $(if $(filter node,$(stack)),COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_INFRA_NODE)),,ARGS COMPOSE_IGNORE_ORPHANS SERVICE))) + $(call make,docker-compose-$(command) STACK="$(stack)" $(if $(filter node,$(stack)),COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_NODE)),,ARGS COMPOSE_IGNORE_ORPHANS SERVICE))) .PHONY: start start: docker-compose-start ## Start application dockers diff --git a/make/apps/def.docker.mk b/make/apps/def.docker.mk index 280f0fc..b8ab18e 100644 --- a/make/apps/def.docker.mk +++ b/make/apps/def.docker.mk @@ -27,7 +27,7 @@ DOCKER_BUILD_VARS ?= APP BRANCH DOCKER_GID DOCKER_REPOSITORY GID G DOCKER_COMPOSE_DOWN_OPTIONS ?= DOCKER_COMPOSE_UP_OPTIONS ?= -d DOCKER_GID ?= $(call getent-group,docker) -DOCKER_IMAGE_TAG ?= $(if $(filter-out $(APP),infra),$(if $(filter $(ENV),$(ENV_DEPLOY)),$(VERSION),$(if $(DRONE_BUILD_NUMBER),$(DRONE_BUILD_NUMBER),latest)),latest) +DOCKER_IMAGE_TAG ?= $(if $(filter-out $(APP),myos),$(if $(filter $(ENV),$(ENV_DEPLOY)),$(VERSION),$(if $(DRONE_BUILD_NUMBER),$(DRONE_BUILD_NUMBER),latest)),latest) DOCKER_IMAGES ?= $(patsubst %/,%,$(patsubst docker/%,%,$(dir $(wildcard docker/*/Dockerfile)))) DOCKER_PLUGIN ?= rexray/s3fs:latest DOCKER_PLUGIN_ARGS ?= $(foreach var,$(DOCKER_PLUGIN_VARS),$(if $(DOCKER_PLUGIN_$(var)),$(var)='$(DOCKER_PLUGIN_$(var))')) @@ -62,7 +62,7 @@ DOCKER_COMPOSE_DOWN_OPTIONS := --rmi all -v DOCKER_COMPOSE_UP_OPTIONS := -d --build DOCKER_BUILD_CACHE := false ENV_SUFFIX := $(DRONE_BUILD_NUMBER) -ifneq ($(APP), infra) +ifneq ($(APP), myos) COMPOSE_PROJECT_NAME := $(USER)_$(ENV)$(ENV_SUFFIX)_$(APP) COMPOSE_SERVICE_NAME := $(subst _,-,$(COMPOSE_PROJECT_NAME)) DOCKER_REPOSITORY := $(USER)/$(ENV)/$(APP) diff --git a/make/apps/deploy.mk b/make/apps/deploy.mk index 9f0628d..cb485a1 100644 --- a/make/apps/deploy.mk +++ b/make/apps/deploy.mk @@ -12,77 +12,14 @@ deploy: app-deploy ## Run post install hooks in the deployed application ## tag and push docker images to docker registry ## run ansible-pull on hosts to pull docker images from the registry ## tag and push docker images as latest to docker registry -deploy@%: infra-base build@% ## Deploy application docker images +deploy@%: myos-base build@% ## Deploy application docker images $(call make,docker-login docker-tag docker-push) - $(call make,infra-ansible-pull@$(ENV) ANSIBLE_DOCKER_IMAGE_TAG=$(VERSION) ANSIBLE_TAGS=aws,,APP AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY) + $(call make,myos-ansible-pull@$(ENV) ANSIBLE_DOCKER_IMAGE_TAG=$(VERSION) ANSIBLE_TAGS=aws,,APP AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY) $(call make,docker-tag-latest docker-push-latest) -.PHONY: deploy-aws-codedeploy-% -deploy-aws-codedeploy-%: - $(call exec,git fetch subrepo/$(SUBREPO)) -ifneq ($(wildcard ../infra),) - $(call make,aws-codedeploy,../infra,CODEDEPLOY_APP_NAME CODEDEPLOY_DEPLOYMENT_CONFIG CODEDEPLOY_DEPLOYMENT_GROUP CODEDEPLOY_DESCRIPTION CODEDEPLOY_GITHUB_REPO CODEDEPLOY_GITHUB_COMMIT_ID) -endif - -.PHONY: deploy-assets-install -deploy-assets-install: - su -s /bin/sh www-data -c "php app/console --no-interaction assets:install --env=prod" - su -s /bin/sh www-data -c "php app/console --no-interaction assetic:dump --env=prod" - -.PHONY: deploy-cache-clear -deploy-cache-clear: - su -s /bin/sh www-data -c "php app/console --no-interaction cache:clear --env=prod" - -.PHONY: deploy-cache-warmup -deploy-cache-warmup: - su -s /bin/sh www-data -c "php app/console --no-interaction cache:warmup --env=prod" - -.PHONY: deploy-composer -deploy-composer: - su -s /bin/sh www-data -c "composer install --prefer-dist --optimize-autoloader --no-progress --no-interaction --no-dev" - -.PHONY: deploy-doctrine-migrations-migrate -deploy-doctrine-migrations-migrate: - su -s /bin/sh www-data -c "php app/console --no-interaction doctrine:migrations:migrate" - -.PHONY: deploy-npm -deploy-npm: deploy-npm-install deploy-npm-run-build - -.PHONY: deploy-npm-install -deploy-npm-install: - npm set progress=false - npm install -s - -.PHONY: deploy-npm-run-build -deploy-npm-run-build: - npm run build:prod - .PHONY: deploy-ping deploy-ping: deploy-ping-slack .PHONY: deploy-ping-slack deploy-ping-slack: curl -X POST --data-urlencode 'payload={"text": "$(DEPLOY_PING_TEXT)"}' $(DEPLOY_SLACK_HOOK) ||: - -.PHONY: deploy-supervisorctl-restart-all -deploy-supervisorctl-restart-all: - supervisorctl restart all - -.PHONY: deploy-supervisorctl-start-all -deploy-supervisorctl-start-all: - supervisorctl start all - -.PHONY: deploy-supervisorctl-stop-all -deploy-supervisorctl-stop-all: - supervisorctl stop all - -.PHONY: deploy-yarn -deploy-yarn: deploy-yarn-install - -.PHONY: deploy-yarn-build -deploy-yarn-build: - yarn build:prod - -.PHONY: deploy-yarn-install -deploy-yarn-install: - yarn install diff --git a/make/apps/docker.mk b/make/apps/docker.mk index ebefdb0..01924d9 100644 --- a/make/apps/docker.mk +++ b/make/apps/docker.mk @@ -2,12 +2,12 @@ # DOCKER .PHONY: docker-build -docker-build: docker-images-infra +docker-build: docker-images-myos $(foreach image,$(or $(SERVICE),$(DOCKER_IMAGES)),$(call make,docker-build-$(image))) .PHONY: docker-build-% docker-build-%: - if grep -q DOCKER_REPOSITORY docker/$*/Dockerfile 2>/dev/null; then $(eval DOCKER_BUILD_ARGS:=$(subst $(DOCKER_REPOSITORY),$(DOCKER_REPOSITORY_INFRA),$(DOCKER_BUILD_ARGS))) true; fi + if grep -q DOCKER_REPOSITORY docker/$*/Dockerfile 2>/dev/null; then $(eval DOCKER_BUILD_ARGS:=$(subst $(DOCKER_REPOSITORY),$(DOCKER_REPOSITORY_MYOS),$(DOCKER_BUILD_ARGS))) true; fi $(if $(wildcard docker/$*/Dockerfile),$(call docker-build,docker/$*)) $(if $(findstring :,$*),$(eval DOCKERFILES := $(wildcard docker/$(subst :,/,$*)/Dockerfile)),$(eval DOCKERFILES := $(wildcard docker/$*/*/Dockerfile))) $(foreach dockerfile,$(DOCKERFILES),$(call docker-build,$(dir $(dockerfile)),$(DOCKER_REPOSITORY)/$(word 2,$(subst /, ,$(dir $(dockerfile)))):$(lastword $(subst /, ,$(dir $(dockerfile)))),"") && true) @@ -27,7 +27,7 @@ docker-commit-%: $(foreach service,$(or $(SERVICE),$(SERVICES)),$(call docker-commit,$(service),,,$*)) .PHONY: docker-compose-build -docker-compose-build: docker-images-infra +docker-compose-build: docker-images-myos $(eval DRYRUN_IGNORE := true) $(eval SERVICES ?= $(shell $(call docker-compose,--log-level critical config --services))) $(eval DRYRUN_IGNORE := false) @@ -66,7 +66,7 @@ docker-compose-ps: $(call docker-compose,ps) .PHONY: docker-compose-rebuild -docker-compose-rebuild: docker-images-infra +docker-compose-rebuild: docker-images-myos $(call make,docker-compose-build DOCKER_BUILD_NO_CACHE=true) .PHONY: docker-compose-recreate @@ -111,18 +111,27 @@ docker-compose-stop: $(call docker-compose,stop $(if $(filter $(SERVICE),$(SERVICES)),$(SERVICE))) .PHONY: docker-compose-up -docker-compose-up: docker-images-infra +docker-compose-up: docker-images-myos $(eval DRYRUN_IGNORE := true) $(eval SERVICES ?= $(shell $(call docker-compose,--log-level critical config --services))) $(eval DRYRUN_IGNORE := false) $(call docker-compose,up $(DOCKER_COMPOSE_UP_OPTIONS) $(if $(filter $(SERVICE),$(SERVICES)),$(SERVICE))) -.PHONY: docker-images-infra -docker-images-infra: - $(foreach image,$(subst $(quote),,$(DOCKER_IMAGES_INFRA)),$(call make,infra-docker-build-$(image))) +.PHONY: docker-images-myos +docker-images-myos: + $(foreach image,$(subst $(quote),,$(DOCKER_IMAGES_MYOS)),$(call make,myos-docker-build-$(image))) + +.PHONY: docker-images-rm +docker-images-rm: + $(call make,docker-images-rm-$(DOCKER_REPOSITORY)/) + +.PHONY: docker-images-rm-% +docker-images-rm-%: + docker images |awk '$$1 ~ /^$(subst /,\/,$*)/ {print $$3}' |sort -u |while read image; do docker rmi -f $$image; done .PHONY: docker-login -docker-login: infra-aws-ecr-login +docker-login: myos-base + $(ECHO) docker login .PHONY: docker-network-create docker-network-create: docker-network-create-$(DOCKER_NETWORK) @@ -175,6 +184,13 @@ docker-rebuild: docker-rebuild-%: $(call make,docker-build-$* DOCKER_BUILD_CACHE=false) +.PHONY: docker-rm +docker-rm: docker-rm-$(COMPOSE_PROJECT_NAME) + +.PHONY: docker-rm-% +docker-rm-%: + docker ps -a |awk '$$NF ~ /^$*/ {print $$NF}' |while read docker; do docker rm -f $$docker; done + .PHONY: docker-run docker-run: SERVICE ?= $(DOCKER_SERVICE) docker-run: @@ -209,3 +225,10 @@ ifneq ($(filter $(DEPLOY),true),) else printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}target${COLOR_RESET} $@ ${COLOR_GREEN}not enabled in${COLOR_RESET} $(APP).\n" >&2 endif + +.PHONY: docker-volume-rm +docker-volume-rm: docker-volume-rm-$(COMPOSE_PROJECT_NAME) + +.PHONY: docker-volume-rm-% +docker-volume-rm-%: + docker volume ls |awk '$$2 ~ /^$*/ {print $$2}' |sort -u |while read volume; do docker volume rm $$volume; done diff --git a/make/apps/infra/clean.mk b/make/apps/infra/clean.mk deleted file mode 100644 index 6b29859..0000000 --- a/make/apps/infra/clean.mk +++ /dev/null @@ -1,18 +0,0 @@ -## -# CLEAN - -.PHONY: clean-docker-% -clean-docker-%: - docker ps -a |awk '$$NF ~ /_$*/ && $$NF !~ /_infra_/ {print $$NF}' |while read docker; do docker rm -f $$docker; done - -.PHONY: clean-elasticsearch-% -clean-elasticsearch-%: - docker ps |awk '$$NF ~ /infra_elasticsearch/' |sed 's/^.*:\([0-9]*\)->9200\/tcp.*$$/\1/' |while read port; do echo -e "DELETE /$* HTTP/1.0\n\n" |nc localhost $$port; done - -.PHONY: clean-images-% -clean-images-%: - docker images |awk '$$1 ~ /\/$*/ && $$1 !~ /\/infra\// {print $$3}' |sort -u |while read image; do docker rmi -f $$image; done - -.PHONY: clean-volumes-% -clean-volumes-%: - docker volume ls |awk '$$2 ~ /_$*/ && $$2 !~ /_infra_/ {print $$2}' |sort -u |while read volume; do docker volume rm $$volume; done diff --git a/make/apps/install.mk b/make/apps/install.mk index 2378e58..4135221 100644 --- a/make/apps/install.mk +++ b/make/apps/install.mk @@ -5,21 +5,18 @@ install: app-install ## Install application .PHONY: install-mysql-database-% -install-mysql-database-%: infra-base +install-mysql-database-%: myos-base $(call exec,mysql -h mysql -u root -proot $* -e "use $*" >/dev/null 2>&1 || mysql -h mysql -u root -proot mysql -e "create database $* character set utf8 collate utf8_unicode_ci;") $(call exec,mysql -h mysql -u $* -p$* $* -e "use $*" >/dev/null 2>&1 || mysql -h mysql -u root -proot mysql -e "grant all privileges on $*.* to '\''$*'\''@'\''%'\'' identified by '\''$*'\''; flush privileges;") $(call exec,[ $$(mysql -h mysql -u $* -p$* $* -e "show tables" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.mysql.gz" ] && gzip -cd "${APP_DIR}/$*.mysql.gz" |mysql -h mysql -u root -proot $* || true) .PHONY: install-pgsql-database-% -install-pgsql-database-%: infra-base +install-pgsql-database-%: myos-base $(call exec,PGPASSWORD=$* psql -h postgres -U $* template1 -c "\q" >/dev/null 2>&1 || PGPASSWORD=postgres psql -h postgres -U postgres -c "create user $* with createdb password '\''$*'\'';") $(call exec,PGPASSWORD=$* psql -h postgres -U $* -d $* -c "" >/dev/null 2>&1 || PGPASSWORD=postgres psql -h postgres -U postgres -c "create database $* owner $* ;") $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql.gz" ] && gzip -cd "${APP_DIR}/$*.pgsql.gz" |PGPASSWORD="postgres" psql -h postgres -U postgres -d $* || true) $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql" ] && PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH SUPERUSER" && PGPASSWORD="postgres" pg_restore -h postgres --no-owner --role=$* -U postgres -d $* ${APP_DIR}/$*.pgsql && PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH NOSUPERUSER" || true) -.PHONY: install-infra -install-infra: infra-install - .PHONY: install-parameters install-parameters: $(call install-parameters) diff --git a/make/def.docker.mk b/make/def.docker.mk index d5088b5..f496021 100644 --- a/make/def.docker.mk +++ b/make/def.docker.mk @@ -1,25 +1,25 @@ COMPOSE_VERSION ?= 1.24.1 -COMPOSE_PROJECT_NAME_INFRA ?= $(USER)_$(ENV)_infra -COMPOSE_PROJECT_NAME_INFRA_NODE ?= node_infra +COMPOSE_PROJECT_NAME_MYOS ?= $(USER)_$(ENV)_myos +COMPOSE_PROJECT_NAME_NODE ?= node DOCKER_EXEC_OPTIONS ?= DOCKER_IMAGE ?= $(DOCKER_IMAGE_CLI) -DOCKER_IMAGE_CLI ?= $(DOCKER_REPOSITORY_INFRA)/cli -DOCKER_IMAGE_SSH ?= $(DOCKER_REPOSITORY_INFRA)/ssh +DOCKER_IMAGE_CLI ?= $(DOCKER_REPOSITORY_MYOS)/cli +DOCKER_IMAGE_SSH ?= $(DOCKER_REPOSITORY_MYOS)/ssh DOCKER_NAME ?= $(DOCKER_NAME_CLI) -DOCKER_NAME_CLI ?= $(COMPOSE_PROJECT_NAME_INFRA)_cli -DOCKER_NAME_SSH ?= $(COMPOSE_PROJECT_NAME_INFRA)_ssh +DOCKER_NAME_CLI ?= $(COMPOSE_PROJECT_NAME_MYOS)_cli +DOCKER_NAME_SSH ?= $(COMPOSE_PROJECT_NAME_MYOS)_ssh DOCKER_NETWORK ?= $(DOCKER_NETWORK_PRIVATE) DOCKER_NETWORK_PRIVATE ?= $(ENV) DOCKER_NETWORK_PUBLIC ?= node -DOCKER_REPOSITORY_INFRA ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_INFRA)) -DOCKER_REPOSITORY_INFRA_NODE ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_INFRA_NODE)) +DOCKER_REPOSITORY_MYOS ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_MYOS)) +DOCKER_REPOSITORY_NODE ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_NODE)) # DOCKER_RUN_OPTIONS: default options to `docker run` command DOCKER_RUN_OPTIONS ?= --rm -it # DOCKER_RUN_VOLUME: options to `docker run` command to mount additionnal volumes DOCKER_RUN_VOLUME ?= -v $$PWD:$$PWD DOCKER_RUN_WORKDIR ?= -w $$PWD -DOCKER_VOLUME_SSH ?= $(COMPOSE_PROJECT_NAME_INFRA)_ssh -ENV_VARS += DOCKER_NETWORK_PRIVATE DOCKER_NETWORK_PUBLIC DOCKER_REPOSITORY_INFRA DOCKER_REPOSITORY_INFRA_NODE DOCKER_VOLUME_SSH +DOCKER_VOLUME_SSH ?= $(COMPOSE_PROJECT_NAME_MYOS)_ssh +ENV_VARS += DOCKER_NETWORK_PRIVATE DOCKER_NETWORK_PUBLIC DOCKER_REPOSITORY_MYOS DOCKER_REPOSITORY_NODE DOCKER_VOLUME_SSH ifeq ($(DRONE), true) DOCKER_RUN_OPTIONS := --rm --network $(DOCKER_NETWORK) diff --git a/make/def.mk b/make/def.mk index 766ea8e..c5c981e 100644 --- a/make/def.mk +++ b/make/def.mk @@ -2,12 +2,12 @@ comma ?= , dollar ?= $ dquote ?= " quote ?= ' -APP ?= $(if $(wildcard .git),$(if $(wildcard */.gitrepo),,$(notdir $(CURDIR))),$(notdir $(CURDIR))) +APP ?= $(if $(wildcard .git),$(notdir $(CURDIR))) APP_DIR ?= $(if $(APP),$(CURDIR)) BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) CMDS ?= exec exec:% exec@% run run:% run@% COMMIT ?= $(shell git rev-parse $(BRANCH) 2>/dev/null) -CONTEXT ?= $(shell awk 'BEGIN {FS="="}; $$1 !~ /^(\#|$$)/ {print $$1}' .env.dist 2>/dev/null) BRANCH ENV_FILE UID USER VERSION +CONTEXT ?= APP BRANCH ENV_FILE $(shell awk 'BEGIN {FS="="}; $$1 !~ /^(\#|$$)/ {print $$1}' .env.dist 2>/dev/null) UID USER VERSION DEBUG ?= false DOCKER ?= true DRONE ?= false @@ -17,14 +17,14 @@ DRYRUN_RECURSIVE ?= false ENV ?= local ENV_DEPLOY ?= preprod prod ENV_FILE ?= $(wildcard ../$(PARAMETERS)/$(ENV)/$(APP)/.env) .env -ENV_LIST ?= local dev tests preprod prod +ENV_LIST ?= local dev tests preprod prod #TODO: staging develop ENV_RESET ?= false ENV_VARS ?= APP APP_DIR BRANCH ENV HOSTNAME GID MONOREPO MONOREPO_DIR TAG UID USER VERSION GID ?= $(shell id -g) GIT_PARAMETERS_REPOSITORY ?= $(call pop,$(GIT_UPSTREAM_REPOSITORY))/$(PARAMETERS) GIT_REPOSITORY ?= $(if $(SUBREPO),$(shell awk -F ' = ' '$$1 ~ /^[[\s\t]]*remote$$/ {print $$2}' .gitrepo),$(shell git config --get remote.origin.url)) GIT_UPSTREAM_REPOSITORY ?= $(if $(findstring ://,$(GIT_REPOSITORY)),$(call pop,$(call pop,$(GIT_REPOSITORY)))/,$(call pop,$(GIT_REPOSITORY),:):)$(GIT_UPSTREAM_USER)/$(lastword $(subst /, ,$(GIT_REPOSITORY))) -GIT_UPSTREAM_USER ?= $(MONOREPO) +GIT_UPSTREAM_USER ?= $(MONOREPO) #TODO HOSTNAME ?= $(shell hostname |sed 's/\..*//') MAKE_ARGS ?= $(foreach var,$(MAKE_VARS),$(if $($(var)),$(var)='$($(var))')) MAKE_VARS ?= ENV @@ -76,19 +76,6 @@ DRYRUN_RECURSIVE := true endif endif -ifeq ($(HOST_SYSTEM),DARWIN) -define getent-group -$(shell dscl . -read /Groups/$(1) 2>/dev/null |awk '$$1 == "PrimaryGroupID:" {print $$2}') -endef -ifneq ($(DOCKER),true) -SED_SUFFIX='\'\'' -endif -else -define getent-group -$(shell getent group $(1) 2>/dev/null |awk -F: '{print $$3}') -endef -endif - define conf $(eval file := $(1)) $(eval block := $(2)) @@ -115,14 +102,20 @@ define conf done < "$(file)" endef +define env + IFS=$$'\n'; $(ECHO) env $(foreach var,$(ENV_VARS),$(if $($(var)),$(var)='$($(var))')) $(shell printenv |awk -F '=' 'NR == FNR { if($$1 !~ /^(\#|$$)/) { A[$$1]; next } } ($$1 in A)' .env.dist - 2>/dev/null) $$(cat $(ENV_FILE) 2>/dev/null |awk -F "=" '$$1 ~! /^\(#|$$\)/') $(1) +endef + define force while true; do [ $$(ps x |awk 'BEGIN {nargs=split("'"$$*"'",args)} $$field == args[1] { matched=1; for (i=1;i<=NF-field;i++) { if ($$(i+field) == args[i+1]) {matched++} } if (matched == nargs) {found++} } END {print found+0}' field=4) -eq 0 ] && $(ECHO) $(1) || sleep 1; done endef -pop = $(patsubst %$(or $(2),/)$(lastword $(subst $(or $(2),/), ,$(1))),%,$(1)) - -define sed -$(call exec,sed -i $(SED_SUFFIX) '\''$(1)'\'' $(2)) +define getent-group +ifeq ($(HOST_SYSTEM),DARWIN) +$(shell dscl . -read /Groups/$(1) 2>/dev/null |awk '$$1 == "PrimaryGroupID:" {print $$2}') +else +$(shell getent group $(1) 2>/dev/null |awk -F: '{print $$3}') +endif endef ## @@ -153,12 +146,23 @@ define make $(if $(filter $(DRYRUN_RECURSIVE),true),$(MAKE) $(MAKE_DIR) $(patsubst %,-o %,$(MAKE_OLDFILE)) MAKE_OLDFILE="$(MAKE_OLDFILE)" DRYRUN=$(DRYRUN) RECURSIVE=$(RECURSIVE) $(MAKE_ARGS) $(cmd)) endef +pop = $(patsubst %$(or $(2),/)$(lastword $(subst $(or $(2),/), ,$(1))),%,$(1)) + +define sed +ifeq ($(HOST_SYSTEM),DARWIN) +ifneq ($(DOCKER),true) +SED_SUFFIX='\'\'' +endif +endif +$(call exec,sed -i $(SED_SUFFIX) '\''$(1)'\'' $(2)) +endef + ifneq ($(MONOREPO),) ifneq ($(wildcard .gitrepo),) -INFRA := ../infra +MYOS := ../myos MAKE_SUBDIRS := subrepo else -INFRA := infra +MYOS := myos MAKE_SUBDIRS := monorepo endif endif @@ -187,12 +191,10 @@ $(foreach env,$(ENV_LIST),$(eval TARGET := %\:$(env)) $(eval ASSIGN_ENV := ENV:= # set ENV=$(env) for each target ending with @$(env) $(foreach env,$(ENV_LIST),$(eval %@$(env): ENV:=$(env))) -# Accept arguments for CMDS targets +# Accept arguments for CMDS targets and turn them into do-nothing targets ifneq ($(filter $(CMDS),$(firstword $(MAKECMDGOALS))),) -# set $ARGS with following arguments ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) ARGS := $(subst :,\:,$(ARGS)) ARGS := $(subst &,\&,$(ARGS)) -# ...and turn them into do-nothing targets $(eval $(ARGS):;@:) endif diff --git a/make/include.mk b/make/include.mk index 72c0fe4..efa3a1e 100644 --- a/make/include.mk +++ b/make/include.mk @@ -2,4 +2,4 @@ MAKE_DIR := $(patsubst %/,%,$(dir $(lastword $(MAKEFILE_L MAKE_FILES := env.mk def.mk include $(wildcard $(patsubst %,$(MAKE_DIR)/%,$(MAKE_FILES))) $(filter-out $(wildcard $(patsubst %,$(MAKE_DIR)/%,include.mk def.*.mk $(MAKE_FILES))),$(wildcard $(MAKE_DIR)/*.mk)) include $(foreach subdir,$(MAKE_SUBDIRS),$(filter-out $(wildcard $(MAKE_DIR)/$(subdir)/def.mk $(MAKE_DIR)/$(subdir)/def.*.mk),$(wildcard $(MAKE_DIR)/$(subdir)/*.mk))) -include $(wildcard *.mk docker/*.mk stack/*.mk) +include $(wildcard *.mk $(filter-out $(wildcard $(MAKE_DIR)/*.mk),$(wildcard */*.mk))) diff --git a/make/infra.mk b/make/infra.mk index 5e52d28..89e3cb9 100644 --- a/make/infra.mk +++ b/make/infra.mk @@ -1,8 +1,8 @@ ## -# INFRA +# MYOS -.PHONY: infra-% -infra-%: ; -ifneq ($(wildcard $(INFRA)),) - $(call make,$*,$(INFRA)) +.PHONY: myos-% +myos-%: ; +ifneq ($(wildcard $(MYOS)),) + $(call make,$*,$(MYOS)) endif diff --git a/make/monorepo/common.mk b/make/monorepo/common.mk index 5c3934e..e740f9d 100644 --- a/make/monorepo/common.mk +++ b/make/monorepo/common.mk @@ -59,7 +59,7 @@ up: $(APPS) ## Create application dockers .PHONY: $(APPS) $(APPS): $(if $(wildcard $@/Makefile), \ - $(call make,-o install-infra $(patsubst apps-%,%,$(MAKECMDGOALS)) STATUS=0,$(patsubst %/,%,$@),ENV_SUFFIX), \ + $(call make,-o install-myos $(patsubst apps-%,%,$(MAKECMDGOALS)) STATUS=0,$(patsubst %/,%,$@),ENV_SUFFIX), \ printf "${COLOR_BROWN}WARNING${COLOR_RESET}: ${COLOR_GREEN}no app available in folder${COLOR_RESET} $@.\n" >&2) # run targets in $(APPS) diff --git a/make/monorepo/def.drone.mk b/make/monorepo/def.drone.mk index 0145cad..a0dbfd2 100644 --- a/make/monorepo/def.drone.mk +++ b/make/monorepo/def.drone.mk @@ -9,7 +9,7 @@ COMMIT_AFTER := $(DRONE_TAG) COMMIT_BEFORE := $(shell git describe --abbrev=0 --tags $(DRONE_TAG)^) endif APPS_IMPACTED := $(shell git diff --name-only $(COMMIT_BEFORE) $(COMMIT_AFTER) 2>/dev/null |awk -F '/' 'NF>1 && !seen[$$1]++ {print $$1}') -# prevent drone to make down in infra -APPS := $(if $(filter-out down,$(MAKECMDGOALS)),$(filter $(INFRA),$(APPS_IMPACTED))) $(sort $(filter-out $(DIRS) $(INFRA),$(APPS_IMPACTED))) +# prevent drone to make down myos +APPS := $(if $(filter-out down,$(MAKECMDGOALS)),$(filter $(MYOS),$(APPS_IMPACTED))) $(sort $(filter-out $(DIRS) $(MYOS),$(APPS_IMPACTED))) CONTEXT += DRONE_BRANCH DRONE_BUILD_EVENT DRONE_BUILD_NUMBER DRONE_COMMIT_AFTER DRONE_COMMIT_AUTHOR DRONE_COMMIT_AUTHOR_EMAIL DRONE_COMMIT_BEFORE DRONE_COMMIT_REF GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME endif diff --git a/make/monorepo/def.mk b/make/monorepo/def.mk index 67378ef..64fc974 100644 --- a/make/monorepo/def.mk +++ b/make/monorepo/def.mk @@ -1,4 +1,4 @@ -APPS ?= $(INFRA) $(sort $(filter-out $(DIRS) $(INFRA), $(patsubst %/,%,$(wildcard */)) )) +APPS ?= $(MYOS) $(sort $(filter-out $(DIRS) $(MYOS), $(patsubst %/,%,$(wildcard */)) )) APPS_NAME ?= $(foreach app,$(APPS),$(or $(shell awk -F '=' '$$1 == "APP" {print $$2}' $(or $(wildcard $(app)/.env),$(wildcard $(app)/.env.$(ENV)),$(app)/.env.dist) 2>/dev/null),$(app))) CMDS += copy master-tag release release-check release-create release-finish subrepo-push update-subrepo CONTEXT += APPS APPS_NAME ENV RELEASE_INSTALL diff --git a/make/monorepo/git.mk b/make/monorepo/git.mk index 8f7bcdd..53da3c5 100644 --- a/make/monorepo/git.mk +++ b/make/monorepo/git.mk @@ -3,7 +3,7 @@ ## Check if monorepo is up to date with subrepo. subrepo-push saves the parent commit in .gitrepo .PHONY: git-diff-subrepo -git-diff-subrepo: infra-base subrepo-check +git-diff-subrepo: myos-base subrepo-check ## Get parent commit in .gitrepo : awk '$1 == "parent" {print $3}' subrepo/.gitrepo ## Get child of parent commit : git rev-list --ancestry-path parent..HEAD |tail -n 1 ## Compare child commit with our tree : git diff --quiet child -- subrepo @@ -12,23 +12,23 @@ git-diff-subrepo: infra-base subrepo-check $(eval DRYRUN_IGNORE := false) .PHONY: git-fetch-subrepo -git-fetch-subrepo: infra-base subrepo-check +git-fetch-subrepo: myos-base subrepo-check $(call exec,git fetch --prune $(REMOTE)) .PHONY: git-stash -git-stash: infra-base git-status +git-stash: myos-base git-status if [ ! $(STATUS) -eq 0 ]; then \ $(call exec,git stash); \ fi .PHONY: git-status -git-status: infra-base +git-status: myos-base $(eval DRYRUN_IGNORE := true) $(eval STATUS := $(shell $(call exec,git status -uno --porcelain 2>/dev/null |wc -l))) $(eval DRYRUN_IGNORE := false) .PHONY: git-unstash -git-unstash: infra-base +git-unstash: myos-base $(eval STATUS ?= 0) if [ ! $(STATUS) -eq 0 ]; then \ $(call exec,git stash pop); \ @@ -36,7 +36,7 @@ git-unstash: infra-base # Create branch $(BRANCH) from upstream/$* branch .PHONY: branch-create-upstream-% -branch-create-upstream-%: infra-base update-upstream +branch-create-upstream-%: myos-base update-upstream $(call exec,git fetch --prune upstream) $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && echo Unable to create $(BRANCH). || git branch $(BRANCH) upstream/$*) $(call exec,[ $$(git ls-remote --heads upstream $(BRANCH) |wc -l) -eq 0 ] && git push upstream $(BRANCH) || echo Unable to create branch $(BRANCH) on remote upstream.) @@ -44,13 +44,13 @@ branch-create-upstream-%: infra-base update-upstream # Delete branch $(BRANCH) .PHONY: branch-delete -branch-delete: infra-base update-upstream +branch-delete: myos-base update-upstream $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && git branch -d $(BRANCH) || echo Unable to delete branch $(BRANCH).) $(foreach remote,upstream, $(call exec,[ $$(git ls-remote --heads $(remote) $(BRANCH) |wc -l) -eq 1 ] && git push $(remote) :$(BRANCH) || echo Unable to delete branch $(BRANCH) on remote $(remote).) &&) true # Merge branch $(BRANCH) into upstream/$* branch .PHONY: branch-merge-upstream-% -branch-merge-upstream-%: infra-base update-upstream +branch-merge-upstream-%: myos-base update-upstream $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1) $(call exec,git checkout $(BRANCH)) $(call exec,git pull --ff-only upstream $(BRANCH)) @@ -62,7 +62,7 @@ branch-merge-upstream-%: infra-base update-upstream # Create $(TAG) tag to reference upstream/$* branch .PHONY: tag-create-upstream-% -tag-create-upstream-%: infra-base update-upstream +tag-create-upstream-%: myos-base update-upstream ifneq ($(words $(TAG)),0) $(call exec,git checkout $*) $(call exec,git pull --tags --prune upstream $*) @@ -76,7 +76,7 @@ endif # Merge tag $(TAG) into upstream/$* branch .PHONY: tag-merge-upstream-% -tag-merge-upstream-%: infra-base update-upstream +tag-merge-upstream-%: myos-base update-upstream ifneq ($(words $(TAG)),0) $(call exec,git fetch --tags -u --prune upstream $*:$*) $(call exec,git checkout $*) diff --git a/make/monorepo/install.mk b/make/monorepo/install.mk index 5347b64..cfe14da 100644 --- a/make/monorepo/install.mk +++ b/make/monorepo/install.mk @@ -1,8 +1,8 @@ ## # INSTALL -.PHONY: install-infra -install-infra: infra-install +.PHONY: install-myos +install-myos: myos-install .PHONY: install-$(SHARED) install-$(SHARED): $(SHARED) diff --git a/make/monorepo/subrepo.mk b/make/monorepo/subrepo.mk index 1781534..8ca8d5b 100644 --- a/make/monorepo/subrepo.mk +++ b/make/monorepo/subrepo.mk @@ -13,13 +13,13 @@ endif ## Delete branch $(BRANCH) on $(SUBREPO) remote .PHONY: subrepo-branch-delete -subrepo-branch-delete: infra-base subrepo-check +subrepo-branch-delete: myos-base subrepo-check ifneq ($(words $(BRANCH)),0) $(call exec,[ $$(git ls-remote --heads $(REMOTE) $(BRANCH) |wc -l) -eq 1 ] && git push $(REMOTE) :$(BRANCH) || echo Unable to delete branch $(BRANCH) on remote $(REMOTE).) endif .PHONY: subrepo-tag-create-% -subrepo-tag-create-%: infra-base subrepo-check git-fetch-subrepo ## Create $(TAG) tag to reference $(REMOTE)/$* branch +subrepo-tag-create-%: myos-base subrepo-check git-fetch-subrepo ## Create $(TAG) tag to reference $(REMOTE)/$* branch ifneq ($(words $(TAG)),0) $(call exec,[ $$(git ls-remote --tags $(REMOTE) $(TAG) |wc -l) -eq 0 ] || git push $(REMOTE) :refs/tags/$(TAG)) $(call exec,git push $(REMOTE) refs/remotes/subrepo/$(SUBREPO)/$*:refs/tags/$(TAG)) @@ -27,7 +27,7 @@ endif ## Push to subrepo. .PHONY: subrepo-push -subrepo-push: infra-base subrepo-check git-fetch-subrepo git-diff-subrepo +subrepo-push: myos-base subrepo-check git-fetch-subrepo git-diff-subrepo # update .gitrepo only on master branch ifeq ($(BRANCH),master) $(eval UPDATE_SUBREPO_OPTIONS += -u) diff --git a/make/monorepo/update.mk b/make/monorepo/update.mk index a594fa2..0a86d69 100644 --- a/make/monorepo/update.mk +++ b/make/monorepo/update.mk @@ -13,7 +13,7 @@ update-$(PARAMETERS): $(PARAMETERS) $(PARAMETERS): SSH_PUBLIC_HOST_KEYS := $(PARAMETERS_REMOTE_HOST) $(SSH_BASTION_HOSTNAME) $(SSH_REMOTE_HOSTS) $(PARAMETERS): MAKE_VARS += SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PRIVATE_IP_RANGE SSH_PUBLIC_HOST_KEYS -$(PARAMETERS): infra-base +$(PARAMETERS): myos-base $(call exec,[ -d $(PARAMETERS) ] && cd $(PARAMETERS) && git pull --quiet || git clone --quiet $(GIT_PARAMETERS_REPOSITORY)) ## Update release version number in .env @@ -23,16 +23,16 @@ update-release: ## Update remotes .PHONY: update-remotes -update-remotes: infra-base +update-remotes: myos-base $(call exec,git fetch --all --prune --tags -u) .PHONY: update-remote-% -update-remote-%: infra-base +update-remote-%: myos-base $(call exec,git fetch --prune --tags -u $*) ## Update subrepos .PHONY: update-subrepos -update-subrepos: infra-base git-stash $(APPS) git-unstash ## Update subrepos +update-subrepos: myos-base git-stash $(APPS) git-unstash ## Update subrepos $(call exec,git push upstream $(BRANCH)) .PHONY: update-subrepo-% @@ -40,8 +40,8 @@ update-subrepo-%: $(if $(wildcard $*/Makefile),$(call make,update-subrepo,$*)) .PHONY: update-upstream -update-upstream: infra-base .git/refs/remotes/upstream/master +update-upstream: myos-base .git/refs/remotes/upstream/master $(call exec,git fetch --tags upstream) -.git/refs/remotes/upstream/master: infra-base +.git/refs/remotes/upstream/master: myos-base $(call exec,git remote add upstream $(GIT_UPSTREAM_REPOSITORY) 2>/dev/null ||:) diff --git a/stack/elastic.mk b/stack/elastic.mk index 891166c..148e5f8 100644 --- a/stack/elastic.mk +++ b/stack/elastic.mk @@ -1 +1,6 @@ elastic ?= elastic/curator elastic/elasticsearch elastic/kibana alpine/sysctl + +.PHONY: elasticsearch-delete-% +elasticsearch-delete-%: + docker ps |awk '$$NF ~ /myos_elasticsearch/' |sed 's/^.*:\([0-9]*\)->9200\/tcp.*$$/\1/' |while read port; do echo -e "DELETE /$* HTTP/1.0\n\n" |nc localhost $$port; done +