diff --git a/Makefile b/Makefile index 9be7b17..4606b97 100644 --- a/Makefile +++ b/Makefile @@ -3,15 +3,27 @@ include make/include.mk ## # APP +.PHONY: app-bootstrap +app-bootstrap: setup-sysctl +ifeq ($(SETUP_NFSD),true) +ifeq ($(HOST_SYSTEM),DARWIN) + $(call setup-nfsd-osx) +endif +endif + +.PHONY: app-build app-build: myos-base install-build-parameters $(call make,docker-compose-build up) $(call make,docker-compose-exec ARGS='rm -Rf /root/.npm /log-buffer/*' SERVICE=logagent) $(call make,docker-commit) -app-deploy: deploy-ping - +.PHONY: app-install app-install: base node up +.PHONY: app-start +app-start: base-ssh-add + +.PHONY: app-tests app-tests: echo ENV: $(env) echo DOCKER_ENV: $(DOCKER_ENV) diff --git a/make/apps/app/app.mk b/make/apps/app/app.mk deleted file mode 100644 index 1745bbe..0000000 --- a/make/apps/app/app.mk +++ /dev/null @@ -1,5 +0,0 @@ -## -# APP - -.PHONY: start-up -start-up: diff --git a/make/apps/app/def.mk b/make/apps/app/def.mk deleted file mode 100644 index ea6bed8..0000000 --- a/make/apps/app/def.mk +++ /dev/null @@ -1,3 +0,0 @@ -BUILD_APP_VARS ?= APP BRANCH COMMIT DEPLOY_SLACK_HOOK ENV VERSION -COMPOSE_IGNORE_ORPHANS ?= false -ENV_VARS += CONSUL_HTTP_TOKEN diff --git a/make/apps/build.mk b/make/apps/build.mk index 4ef9396..0822854 100644 --- a/make/apps/build.mk +++ b/make/apps/build.mk @@ -19,7 +19,7 @@ build@%: myos-base .PHONY: build-env build-env: SERVICE ?= $(DOCKER_SERVICE) build-env: bootstrap - $(call docker-compose-exec,$(SERVICE),rm -f .env && make .env ENV=$(ENV) && echo BUILD=true >> .env && echo BUILD_DATE='"\'"'$(shell date "+%d/%m/%Y %H:%M:%S %z" 2>/dev/null)'"\'"' >> .env && echo BUILD_STATUS='"\'"'$(shell git status -uno --porcelain 2>/dev/null)'"\'"' >> .env && echo DOCKER=false >> .env && $(foreach var,$(BUILD_APP_VARS),$(if $($(var)),sed -i '/^$(var)=/d' .env && echo $(var)='$($(var))' >> .env &&)) true) + $(call docker-compose-exec,$(SERVICE),rm -f .env && make .env ENV=$(ENV) && echo BUILD=true >> .env && echo BUILD_DATE='"\'"'$(shell date "+%d/%m/%Y %H:%M:%S %z" 2>/dev/null)'"\'"' >> .env && echo BUILD_STATUS='"\'"'$(shell git status -uno --porcelain 2>/dev/null)'"\'"' >> .env && echo DOCKER=false >> .env && $(foreach var,$(BUILD_ENV_VARS),$(if $($(var)),sed -i '/^$(var)=/d' .env && echo $(var)='$($(var))' >> .env &&)) true) # target build-init: Empty build directory .PHONY: build-init diff --git a/make/apps/common.mk b/make/apps/common.mk index c997bfc..547ff6b 100644 --- a/make/apps/common.mk +++ b/make/apps/common.mk @@ -51,7 +51,7 @@ logs: docker-compose-logs ## Display application dockers logs ps: docker-compose-ps ## List application dockers .PHONY: recreate -recreate: docker-compose-recreate start-up ## Recreate application dockers +recreate: docker-compose-recreate app-start ## Recreate application dockers .PHONY: reinstall reinstall: clean ## Reinstall application @@ -59,7 +59,7 @@ reinstall: clean ## Reinstall application $(call make,install) .PHONY: restart -restart: docker-compose-restart start-up ## Restart application +restart: docker-compose-restart app-start ## Restart application .PHONY: run run: ## Run a command in a new docker @@ -108,7 +108,7 @@ stop: docker-compose-stop ## Stop application dockers tests: app-tests ## Test application .PHONY: up -up: docker-compose-up start-up ## Create application dockers +up: docker-compose-up app-start ## Create application dockers .PHONY: update app-update update: app-update ## Update application diff --git a/make/apps/def.deploy.mk b/make/apps/def.deploy.mk index d75c996..45539bf 100644 --- a/make/apps/def.deploy.mk +++ b/make/apps/def.deploy.mk @@ -1,34 +1,5 @@ -CODEDEPLOY_APP_NAME ?= $(APP) -CODEDEPLOY_DEPLOYMENT_GROUP ?= $(CODEDEPLOY_APP_NAME)_$(ENV) -CODEDEPLOY_DEPLOYMENT_CONFIG ?= CodeDeployDefault.AllAtOnce -CODEDEPLOY_DESCRIPTION ?= deploy $(ENV) $(APP) branch: $(BRANCH) commit: $(SUBREPO_COMMIT) tag: $(TAG) version: $(VERSION) -CODEDEPLOY_GITHUB_REPO ?= $(patsubst ssh://git@github.com/%,%,$(GIT_REPOSITORY)) -CODEDEPLOY_GITHUB_COMMIT_ID ?= $(SUBREPO_COMMIT) DEPLOY ?= false -DEPLOY_PING_TEXT ?= app: *$(APP)* branch: *$(BRANCH)* env: *$(ENV)* version: *$(VERSION)* container: *$(CONTAINER)* host: *$(HOST)* -DEPLOY_SLACK_HOOK ?= https://hooks.slack.com/services/123456789/123456789/ABCDEFGHIJKLMNOPQRSTUVWX -HASH ?= $(shell date +%s) +DEPLOY_HASH ?= $(shell date +%s) +DEPLOY_HOOK_TEXT ?= app: *$(APP)* branch: *$(BRANCH)* env: *$(ENV)* version: *$(VERSION)* container: *$(CONTAINER)* host: *$(HOST)* +DEPLOY_HOOK_URL ?= https://hooks.slack.com/services/123456789/123456789/ABCDEFGHIJKLMNOPQRSTUVWX SERVER_NAME ?= $(DOCKER_REGISTRY_USERNAME).$(ENV).$(APP) -OXA_SERVER_NAME ?= web - -ifeq ($(APP),marketplace) -CODEDEPLOY_APP_NAME := front -EC2_SERVER_NAME := front -else ifeq ($(APP),medias) -CODEDEPLOY_APP_NAME := apimedia -EC2_SERVER_NAME := api -else ifeq ($(APP),partners) -CODEDEPLOY_APP_NAME := api -EC2_SERVER_NAME := api -else ifeq ($(APP),workers) -CODEDEPLOY_APP_NAME := worker -EC2_SERVER_NAME := worker -endif -ifeq ($(ENV), preprod) -CODEDEPLOY_APP_NAME := $(CODEDEPLOY_APP_NAME)pp -CODEDEPLOY_DEPLOYMENT_GROUP := $(CODEDEPLOY_APP_NAME)_preprod -OXA_SERVER_NAME := $(EC2_SERVER_NAME)pp-0x -else ifeq ($(ENV), prod) -CODEDEPLOY_DEPLOYMENT_GROUP := $(CODEDEPLOY_APP_NAME)_prod -OXA_SERVER_NAME := $(EC2_SERVER_NAME)-xx -endif diff --git a/make/apps/def.mk b/make/apps/def.mk index cfd6070..71f6e08 100644 --- a/make/apps/def.mk +++ b/make/apps/def.mk @@ -1,4 +1,12 @@ -ENV_VARS += NFS_CONFIG +BUILD_ENV_VARS ?= APP BRANCH COMMIT DEPLOY_HOOK_URL ENV VERSION +COMPOSE_IGNORE_ORPHANS ?= false +ENV_VARS += CONSUL_HTTP_TOKEN GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME NFS_CONFIG +GIT_AUTHOR_EMAIL ?= $(shell git config user.email 2>/dev/null) +GIT_AUTHOR_NAME ?= $(shell git config user.name 2>/dev/null) MOUNT_NFS ?= false NFS_CONFIG ?= addr=$(NFS_HOST),actimeo=3,intr,noacl,noatime,nocto,nodiratime,nolock,soft,rsize=32768,wsize=32768,tcp,rw,vers=3 NFS_HOST ?= host.docker.internal +SETUP_NFSD ?= false +SETUP_NFSD_OSX_CONFIG ?= nfs.server.bonjour=0 nfs.server.mount.regular_files=1 nfs.server.mount.require_resv_port=0 nfs.server.nfsd_threads=16 nfs.server.async=1 +SETUP_SYSCTL ?= false +SETUP_SYSCTL_CONFIG ?= vm.max_map_count=262144 vm.overcommit_memory=1 fs.file-max=8388608 net.core.somaxconn=1024 diff --git a/make/apps/deploy.mk b/make/apps/deploy.mk index cb485a1..6b71f9d 100644 --- a/make/apps/deploy.mk +++ b/make/apps/deploy.mk @@ -1,11 +1,10 @@ -########## -# DEPLOY # -########## +## +# DEPLOY .PHONY: deploy app-deploy # target deploy: Run post install hooks in the deployed application ## Called by ansible after creation of the docker application on remote host -deploy: app-deploy ## Run post install hooks in the deployed application +deploy: app-deploy deploy-ping ## Run post install hooks in the deployed application .PHONY: deploy@% # target deploy@%: Deploy application docker images @@ -18,8 +17,8 @@ deploy@%: myos-base build@% ## Deploy application docker images $(call make,docker-tag-latest docker-push-latest) .PHONY: deploy-ping -deploy-ping: deploy-ping-slack +deploy-ping: deploy-ping-hook -.PHONY: deploy-ping-slack -deploy-ping-slack: - curl -X POST --data-urlencode 'payload={"text": "$(DEPLOY_PING_TEXT)"}' $(DEPLOY_SLACK_HOOK) ||: +.PHONY: deploy-ping-hook +deploy-ping-hook: + curl -X POST --data-urlencode 'payload={"text": "$(DEPLOY_HOOK_TEXT)"}' $(DEPLOY_HOOK_URL) ||: diff --git a/make/apps/git.mk b/make/apps/git.mk new file mode 100644 index 0000000..5ee4a33 --- /dev/null +++ b/make/apps/git.mk @@ -0,0 +1,71 @@ +## +# GIT + +# Create branch $(BRANCH) from upstream/$* branch +.PHONY: git-branch-create-upstream-% +git-branch-create-upstream-%: myos-base update-upstream + $(call exec,git fetch --prune upstream) + $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && echo Unable to create $(BRANCH). || git branch $(BRANCH) upstream/$*) + $(call exec,[ $$(git ls-remote --heads upstream $(BRANCH) |wc -l) -eq 0 ] && git push upstream $(BRANCH) || echo Unable to create branch $(BRANCH) on remote upstream.) + $(call exec,git checkout $(BRANCH)) + +# Delete branch $(BRANCH) +.PHONY: git-branch-delete +git-branch-delete: myos-base update-upstream + $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && git branch -d $(BRANCH) || echo Unable to delete branch $(BRANCH).) + $(foreach remote,upstream, $(call exec,[ $$(git ls-remote --heads $(remote) $(BRANCH) |wc -l) -eq 1 ] && git push $(remote) :$(BRANCH) || echo Unable to delete branch $(BRANCH) on remote $(remote).) &&) true + +# Merge branch $(BRANCH) into upstream/$* branch +.PHONY: git-branch-merge-upstream-% +git-branch-merge-upstream-%: myos-base update-upstream + $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1) + $(call exec,git checkout $(BRANCH)) + $(call exec,git pull --ff-only upstream $(BRANCH)) + $(call exec,git push upstream $(BRANCH)) + $(call exec,git checkout $*) + $(call exec,git pull --ff-only upstream $*) + $(call exec,git merge --no-ff --no-edit $(BRANCH)) + $(call exec,git push upstream $*) + +.PHONY: git-stash +git-stash: myos-base git-status + if [ ! $(STATUS) -eq 0 ]; then \ + $(call exec,git stash); \ + fi + +.PHONY: git-status +git-status: myos-base + $(eval DRYRUN_IGNORE := true) + $(eval STATUS := $(shell $(call exec,git status -uno --porcelain 2>/dev/null |wc -l))) + $(eval DRYRUN_IGNORE := false) + +# Create $(TAG) tag to reference upstream/$* branch +.PHONY: git-tag-create-upstream-% +git-tag-create-upstream-%: myos-base update-upstream +ifneq ($(words $(TAG)),0) + $(call exec,git checkout $*) + $(call exec,git pull --tags --prune upstream $*) + $(call sed,s/^##\? $(TAG).*/## $(TAG) - $(shell date +%Y-%m-%d)/,CHANGELOG.md) + $(call exec,[ $$(git diff CHANGELOG.md 2>/dev/null |wc -l) -eq 0 ] || git commit -m "$$(cat CHANGELOG.md |sed -n '\''/$(TAG)/,/^$$/{s/##\(.*\)/release\1\n/;p;}'\'')" CHANGELOG.md) + $(call exec,[ $$(git tag -l $(TAG) |wc -l) -eq 0 ] || git tag -d $(TAG)) + $(call exec,git tag $(TAG)) + $(call exec,[ $$(git ls-remote --tags upstream $(TAG) |wc -l) -eq 0 ] || git push upstream :refs/tags/$(TAG)) + $(call exec,git push --tags upstream $*) +endif + +# Merge tag $(TAG) into upstream/$* branch +.PHONY: git-tag-merge-upstream-% +git-tag-merge-upstream-%: myos-base update-upstream +ifneq ($(words $(TAG)),0) + $(call exec,git fetch --tags -u --prune upstream $*:$*) + $(call exec,git checkout $*) + $(call exec,git merge --ff --no-edit $(TAG)) + $(call exec,git push upstream $*) +endif + +.PHONY: git-unstash +git-unstash: myos-base + $(eval STATUS ?= 0) + if [ ! $(STATUS) -eq 0 ]; then \ + $(call exec,git stash pop); \ + fi diff --git a/make/apps/infra/bootstrap.mk b/make/apps/infra/bootstrap.mk deleted file mode 100644 index 53208b6..0000000 --- a/make/apps/infra/bootstrap.mk +++ /dev/null @@ -1,10 +0,0 @@ -## -# BOOTSTRAP - -.PHONY: bootstrap-infra -bootstrap-infra: setup-sysctl -ifeq ($(SETUP_NFSD),true) -ifeq ($(HOST_SYSTEM),DARWIN) - $(call setup-nfsd-osx) -endif -endif diff --git a/make/apps/infra/def.mk b/make/apps/infra/def.mk deleted file mode 100644 index 1b3e57c..0000000 --- a/make/apps/infra/def.mk +++ /dev/null @@ -1,36 +0,0 @@ -CMDS += openstack ssh-run terraform -COMPOSE_IGNORE_ORPHANS ?= true -CONTEXT += GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME -DOCKER_BUILD_VARS += SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PUBLIC_HOST_KEYS SSH_PRIVATE_IP_RANGE -DOCKER_SERVICE ?= cli -ELASTICSEARCH_HOST ?= elasticsearch -ELASTICSEARCH_PORT ?= 9200 -ELASTICSEARCH_PROTOCOL ?= http -ENV_VARS += COMPOSE_IGNORE_ORPHANS DOCKER_IMAGE_CLI DOCKER_IMAGE_SSH DOCKER_NAME_CLI DOCKER_NAME_SSH ELASTICSEARCH_HOST ELASTICSEARCH_PASSWORD ELASTICSEARCH_PORT ELASTICSEARCH_PROTOCOL ELASTICSEARCH_USERNAME GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME SETUP_SYSCTL_CONFIG SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PUBLIC_HOST_KEYS SSH_PRIVATE_IP_RANGE -GIT_AUTHOR_EMAIL ?= $(shell git config user.email 2>/dev/null) -GIT_AUTHOR_NAME ?= $(shell git config user.name 2>/dev/null) -HOME ?= /home/$(USER) -NFS_DISK ?= $(NFS_HOST):/$(SHARED) -NFS_OPTIONS ?= rw,rsize=8192,wsize=8192,bg,hard,intr,nfsvers=3,noatime,nodiratime,actimeo=3 -NFS_PATH ?= /srv/$(subst :,,$(NFS_DISK)) -SETUP_NFSD ?= false -SETUP_NFSD_OSX_CONFIG ?= nfs.server.bonjour=0 nfs.server.mount.regular_files=1 nfs.server.mount.require_resv_port=0 nfs.server.nfsd_threads=16 nfs.server.async=1 -SETUP_SYSCTL ?= false -SETUP_SYSCTL_CONFIG ?= vm.max_map_count=262144 vm.overcommit_memory=1 fs.file-max=8388608 net.core.somaxconn=1024 -SHELL ?= /bin/sh -SSH_BASTION_HOSTNAME ?= -SSH_BASTION_USERNAME ?= -SSH_PUBLIC_HOST_KEYS ?= $(SSH_REMOTE_HOSTS) $(SSH_BASTION_HOSTNAME) -SSH_PRIVATE_IP_RANGE ?= 10.10.* -SSH_REMOTE_HOSTS ?= github.com gitlab.com -STACK ?= base logs services - -define setup-nfsd-osx - $(eval dir:=$(or $(1),$(MONOREPO_DIR))) - $(eval uid:=$(or $(2),$(UID))) - $(eval gid:=$(or $(3),$(GID))) - grep "$(dir)" /etc/exports >/dev/null 2>&1 || echo "$(dir) -alldirs -mapall=$(uid):$(gid) localhost" |sudo tee -a /etc/exports >/dev/null - $(foreach config,$(SETUP_NFSD_OSX_CONFIG),grep "$(config)" /etc/nfs.conf >/dev/null 2>&1 || echo "$(config)" |sudo tee -a /etc/nfs.conf >/dev/null &&) true - nfsd status >/dev/null || sudo nfsd enable - showmount -e localhost |grep "$(dir)" >/dev/null 2>&1 || sudo nfsd restart -endef diff --git a/make/apps/infra/infra.mk b/make/apps/infra/infra.mk deleted file mode 100644 index 0ec67f6..0000000 --- a/make/apps/infra/infra.mk +++ /dev/null @@ -1,11 +0,0 @@ -## -# INFRA - -.PHONY: setup-sysctl -setup-sysctl: -ifeq ($(SETUP_SYSCTL),true) - $(foreach config,$(SETUP_SYSCTL_CONFIG),$(call docker-run,--privileged alpine:latest,sysctl -q -w $(config)) &&) true -endif - -.PHONY: start-up -start-up: base-ssh-add diff --git a/make/apps/myos/def.mk b/make/apps/myos/def.mk new file mode 100644 index 0000000..398ad77 --- /dev/null +++ b/make/apps/myos/def.mk @@ -0,0 +1,11 @@ +CMDS += ssh-run +COMPOSE_IGNORE_ORPHANS := true +CONTEXT += GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME +DOCKER_SERVICE ?= cli +ENV_VARS += COMPOSE_IGNORE_ORPHANS DOCKER_IMAGE_CLI DOCKER_IMAGE_SSH DOCKER_NAME_CLI DOCKER_NAME_SSH +HOME ?= /home/$(USER) +NFS_DISK ?= $(NFS_HOST):/$(SHARED) +NFS_OPTIONS ?= rw,rsize=8192,wsize=8192,bg,hard,intr,nfsvers=3,noatime,nodiratime,actimeo=3 +NFS_PATH ?= /srv/$(subst :,,$(NFS_DISK)) +SHELL ?= /bin/sh +STACK ?= base diff --git a/make/apps/infra/def.ssh.mk b/make/apps/myos/def.ssh.mk similarity index 53% rename from make/apps/infra/def.ssh.mk rename to make/apps/myos/def.ssh.mk index 4d3898e..ae4b57f 100644 --- a/make/apps/infra/def.ssh.mk +++ b/make/apps/myos/def.ssh.mk @@ -1,3 +1,12 @@ +DOCKER_BUILD_VARS += $(SSH_ENV_VARS) +ENV_VARS += $(SSH_ENV_VARS) +SSH_BASTION_HOSTNAME ?= +SSH_BASTION_USERNAME ?= +SSH_ENV_VARS ?= SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PUBLIC_HOST_KEYS SSH_PRIVATE_IP_RANGE +SSH_PUBLIC_HOST_KEYS ?= $(SSH_REMOTE_HOSTS) $(SSH_BASTION_HOSTNAME) +SSH_PRIVATE_IP_RANGE ?= 10.10.* +SSH_REMOTE_HOSTS ?= github.com gitlab.com + define ssh-connect $(eval hosts := $(1)) $(eval command := $(2)) diff --git a/make/apps/infra/ssh.mk b/make/apps/myos/ssh.mk similarity index 100% rename from make/apps/infra/ssh.mk rename to make/apps/myos/ssh.mk diff --git a/make/apps/php/def.mk b/make/apps/php/def.mk index 8cb1c7c..9d0afa7 100644 --- a/make/apps/php/def.mk +++ b/make/apps/php/def.mk @@ -1,4 +1,4 @@ -BUILD_APP_VARS += SYMFONY_ENV +BUILD_ENV_VARS += SYMFONY_ENV DOCKER_SERVICE ?= php ifneq (,$(filter $(ENV),$(ENV_DEPLOY))) diff --git a/make/apps/setup.mk b/make/apps/setup.mk new file mode 100644 index 0000000..a686c0f --- /dev/null +++ b/make/apps/setup.mk @@ -0,0 +1,18 @@ +## +# SETUP + +.PHONY: setup-sysctl +setup-sysctl: +ifeq ($(SETUP_SYSCTL),true) + $(foreach config,$(SETUP_SYSCTL_CONFIG),$(call docker-run,--privileged alpine:latest,sysctl -q -w $(config)) &&) true +endif + +define setup-nfsd-osx + $(eval dir:=$(or $(1),$(MONOREPO_DIR))) + $(eval uid:=$(or $(2),$(UID))) + $(eval gid:=$(or $(3),$(GID))) + grep "$(dir)" /etc/exports >/dev/null 2>&1 || echo "$(dir) -alldirs -mapall=$(uid):$(gid) localhost" |sudo tee -a /etc/exports >/dev/null + $(foreach config,$(SETUP_NFSD_OSX_CONFIG),grep "$(config)" /etc/nfs.conf >/dev/null 2>&1 || echo "$(config)" |sudo tee -a /etc/nfs.conf >/dev/null &&) true + nfsd status >/dev/null || sudo nfsd enable + showmount -e localhost |grep "$(dir)" >/dev/null 2>&1 || sudo nfsd restart +endef diff --git a/make/subrepo/subrepo.mk b/make/apps/subrepo/subrepo.mk similarity index 78% rename from make/subrepo/subrepo.mk rename to make/apps/subrepo/subrepo.mk index 1e86d95..5dcc276 100644 --- a/make/subrepo/subrepo.mk +++ b/make/apps/subrepo/subrepo.mk @@ -13,11 +13,3 @@ subrepos-tag-create-%: .PHONY: subrepos-push subrepos-push: $(call make,subrepo-push,..,SUBREPO BRANCH) - -.PHONY: git-stash -git-stash: - $(call make,git-stash,..) - -.PHONY: git-unstash -git-unstash: - $(call make,git-unstash,..) diff --git a/make/def.mk b/make/def.mk index 7ff20e8..9bcd31f 100644 --- a/make/def.mk +++ b/make/def.mk @@ -7,8 +7,11 @@ APP_DIR ?= $(if $(APP),$(CURDIR)) APP_DOMAIN ?= $(if $(APP),$(ENV)$(addprefix .,$(DOMAIN))) APP_HOST ?= $(if $(APP),$(APP)$(addprefix .,$(APP_DOMAIN))) APP_NAME ?= $(APP) +APP_PARAMETERS_REPOSITORY ?= $(GIT_PARAMETERS_REPOSITORY) APP_PATH ?= /$(APP_PATH_PREFIX) APP_SCHEME ?= https +APP_TYPE ?= $(if $(SUBREPO),subrepo) $(if $(MYOS),,myos) +APP_UPSTREAM_REPOSITORY ?= $(GIT_UPSTREAM_REPOSITORY) APP_URI ?= $(if $(APP),$(APP_HOST)$(APP_PATH)) APP_URL ?= $(if $(APP),$(APP_SCHEME)://$(APP_URI)) APPS ?= $(if $(MONOREPO),$(sort $(patsubst $(MONOREPO_DIR)/%/.git,%,$(wildcard $(MONOREPO_DIR)/*/.git)))) @@ -26,9 +29,9 @@ DRYRUN ?= false DRYRUN_IGNORE ?= false DRYRUN_RECURSIVE ?= false ENV ?= dist -ENV_DEPLOY ?= preprod prod +ENV_DEPLOY ?= $(if $(APP),$(shell ls .git/refs/heads/)) ENV_FILE ?= $(wildcard ../$(PARAMETERS)/$(ENV)/$(APP)/.env) .env -ENV_LIST ?= local dev tests preprod prod #TODO: staging develop +ENV_LIST ?= debug local tests release master #TODO: staging develop ENV_RESET ?= false ENV_VARS ?= APP APP_DIR APP_DOMAIN BRANCH ENV HOSTNAME GID MONOREPO MONOREPO_DIR TAG UID USER VERSION GID ?= $(shell id -g 2>/dev/null) @@ -37,9 +40,9 @@ GIT_REPOSITORY ?= $(if $(SUBREPO),$(shell awk -F ' = ' '$$1 ~ / GIT_UPSTREAM_REPOSITORY ?= $(if $(findstring ://,$(GIT_REPOSITORY)),$(call pop,$(call pop,$(GIT_REPOSITORY)))/,$(call pop,$(GIT_REPOSITORY),:):)$(GIT_UPSTREAM_USER)/$(lastword $(subst /, ,$(GIT_REPOSITORY))) GIT_UPSTREAM_USER ?= $(or $(MONOREPO),$(USER)) HOSTNAME ?= $(shell hostname 2>/dev/null |sed 's/\..*//') -MAKE_ARGS ?= $(foreach var,$(MAKE_VARS),$(if $($(var)),$(var)='$($(var))')) -MAKE_VARS ?= ENV -MAKE_SUBDIRS ?= $(if $(filter myos,$(MYOS)),monorepo,$(if $(SUBREPO),subrepo )$(if $(APP),apps $(foreach type,$(APP_TYPE),$(if $(wildcard $(MAKE_DIR)/apps/$(type)),apps/$(type))))) +MAKE_ENV_ARGS ?= $(foreach var,$(MAKE_ENV_VARS),$(if $($(var)),$(var)='$($(var))')) +MAKE_ENV_VARS ?= ENV +MAKE_SUBDIRS ?= $(if $(filter myos,$(MYOS)),monorepo,$(if $(APP),apps $(foreach type,$(APP_TYPE),$(if $(wildcard $(MAKE_DIR)/apps/$(type)),apps/$(type))))) MAKECMDARGS ?= $(foreach var,$(MAKECMDVARS),$(var)='$($(var))') MAKECMDVARS ?= $(strip $(foreach var, $(filter-out .VARIABLES,$(.VARIABLES)), $(if $(filter command\ line,$(origin $(var))),$(var)))) MAKEENVVARS ?= $(strip $(foreach var, $(filter-out .VARIABLES,$(.VARIABLES)), $(if $(filter environment,$(origin $(var))),$(var)))) @@ -140,8 +143,8 @@ gid = $(shell grep '^$(1):' /etc/group 2>/dev/null |awk -F: '{print $$3}') # 2nd arg: directory to call make from # 3rd arg: list of variables to pass to make (ENV by default) # 4th arg: path to .env file with additional arguments to call make with (file must exist when calling make) - # add list of VARIABLE=VALUE from vars to MAKE_ARGS - # add list of arguments from file to MAKE_ARGS + # add list of VARIABLE=VALUE from vars to MAKE_ENV_ARGS + # add list of arguments from file to MAKE_ENV_ARGS # eval MAKE_DIR option to -C $(2) if $(2) given # add current target to MAKE_OLDFILE (list of already fired targets) # print command that will be run if VERBOSE mode @@ -152,13 +155,13 @@ define make $(eval dir := $(2)) $(eval vars := $(3)) $(eval file := $(4)) - $(if $(vars),$(eval MAKE_ARGS += $(foreach var,$(vars),$(if $($(var)),$(var)='$($(var))')))) - $(if $(wildcard $(file)),$(eval MAKE_ARGS += $(shell cat $(file) |sed '/^$$/d; /^#/d; /=/!d; s/^[[\s\t]]*//; s/[[\s\t]]*=[[\s\t]]*/=/;' |awk -F '=' '{print $$1"='\''"$$2"'\''"}'))) + $(if $(vars),$(eval MAKE_ENV_ARGS += $(foreach var,$(vars),$(if $($(var)),$(var)='$($(var))')))) + $(if $(wildcard $(file)),$(eval MAKE_ENV_ARGS += $(shell cat $(file) |sed '/^$$/d; /^#/d; /=/!d; s/^[[\s\t]]*//; s/[[\s\t]]*=[[\s\t]]*/=/;' |awk -F '=' '{print $$1"='\''"$$2"'\''"}'))) $(eval MAKE_DIR := $(if $(dir),-C $(dir))) $(eval MAKE_OLDFILE += $(filter-out $(MAKE_OLDFILE), $^)) - $(if $(filter $(VERBOSE),true),printf '${COLOR_GREEN}Running${COLOR_RESET} "'"make $(MAKE_ARGS) $(cmd)"'" $(if $(dir),${COLOR_BLUE}in folder${COLOR_RESET} $(dir) )\n') - $(ECHO) $(MAKE) $(MAKE_DIR) $(patsubst %,-o %,$(MAKE_OLDFILE)) MAKE_OLDFILE="$(MAKE_OLDFILE)" $(MAKE_ARGS) $(cmd) - $(if $(filter $(DRYRUN_RECURSIVE),true),$(MAKE) $(MAKE_DIR) $(patsubst %,-o %,$(MAKE_OLDFILE)) MAKE_OLDFILE="$(MAKE_OLDFILE)" DRYRUN=$(DRYRUN) RECURSIVE=$(RECURSIVE) $(MAKE_ARGS) $(cmd)) + $(if $(filter $(VERBOSE),true),printf '${COLOR_GREEN}Running${COLOR_RESET} "'"make $(MAKE_ENV_ARGS) $(cmd)"'" $(if $(dir),${COLOR_BLUE}in folder${COLOR_RESET} $(dir) )\n') + $(ECHO) $(MAKE) $(MAKE_DIR) $(patsubst %,-o %,$(MAKE_OLDFILE)) MAKE_OLDFILE="$(MAKE_OLDFILE)" $(MAKE_ENV_ARGS) $(cmd) + $(if $(filter $(DRYRUN_RECURSIVE),true),$(MAKE) $(MAKE_DIR) $(patsubst %,-o %,$(MAKE_OLDFILE)) MAKE_OLDFILE="$(MAKE_OLDFILE)" DRYRUN=$(DRYRUN) RECURSIVE=$(RECURSIVE) $(MAKE_ENV_ARGS) $(cmd)) endef pop = $(patsubst %$(or $(2),/)$(lastword $(subst $(or $(2),/), ,$(1))),%,$(1)) diff --git a/make/monorepo/git.mk b/make/monorepo/git.mk index 53da3c5..b472d7b 100644 --- a/make/monorepo/git.mk +++ b/make/monorepo/git.mk @@ -1,7 +1,7 @@ ## # GIT -## Check if monorepo is up to date with subrepo. subrepo-push saves the parent commit in .gitrepo +## Check if monorepo is up to date with subrepo. subrepo-push saves the parent commit in file subrepo/.gitrepo .PHONY: git-diff-subrepo git-diff-subrepo: myos-base subrepo-check ## Get parent commit in .gitrepo : awk '$1 == "parent" {print $3}' subrepo/.gitrepo @@ -14,72 +14,3 @@ git-diff-subrepo: myos-base subrepo-check .PHONY: git-fetch-subrepo git-fetch-subrepo: myos-base subrepo-check $(call exec,git fetch --prune $(REMOTE)) - -.PHONY: git-stash -git-stash: myos-base git-status - if [ ! $(STATUS) -eq 0 ]; then \ - $(call exec,git stash); \ - fi - -.PHONY: git-status -git-status: myos-base - $(eval DRYRUN_IGNORE := true) - $(eval STATUS := $(shell $(call exec,git status -uno --porcelain 2>/dev/null |wc -l))) - $(eval DRYRUN_IGNORE := false) - -.PHONY: git-unstash -git-unstash: myos-base - $(eval STATUS ?= 0) - if [ ! $(STATUS) -eq 0 ]; then \ - $(call exec,git stash pop); \ - fi - -# Create branch $(BRANCH) from upstream/$* branch -.PHONY: branch-create-upstream-% -branch-create-upstream-%: myos-base update-upstream - $(call exec,git fetch --prune upstream) - $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && echo Unable to create $(BRANCH). || git branch $(BRANCH) upstream/$*) - $(call exec,[ $$(git ls-remote --heads upstream $(BRANCH) |wc -l) -eq 0 ] && git push upstream $(BRANCH) || echo Unable to create branch $(BRANCH) on remote upstream.) - $(call exec,git checkout $(BRANCH)) - -# Delete branch $(BRANCH) -.PHONY: branch-delete -branch-delete: myos-base update-upstream - $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1 && git branch -d $(BRANCH) || echo Unable to delete branch $(BRANCH).) - $(foreach remote,upstream, $(call exec,[ $$(git ls-remote --heads $(remote) $(BRANCH) |wc -l) -eq 1 ] && git push $(remote) :$(BRANCH) || echo Unable to delete branch $(BRANCH) on remote $(remote).) &&) true - -# Merge branch $(BRANCH) into upstream/$* branch -.PHONY: branch-merge-upstream-% -branch-merge-upstream-%: myos-base update-upstream - $(call exec,git rev-parse --verify $(BRANCH) >/dev/null 2>&1) - $(call exec,git checkout $(BRANCH)) - $(call exec,git pull --ff-only upstream $(BRANCH)) - $(call exec,git push upstream $(BRANCH)) - $(call exec,git checkout $*) - $(call exec,git pull --ff-only upstream $*) - $(call exec,git merge --no-ff --no-edit $(BRANCH)) - $(call exec,git push upstream $*) - -# Create $(TAG) tag to reference upstream/$* branch -.PHONY: tag-create-upstream-% -tag-create-upstream-%: myos-base update-upstream -ifneq ($(words $(TAG)),0) - $(call exec,git checkout $*) - $(call exec,git pull --tags --prune upstream $*) - $(call sed,s/^##\? $(TAG).*/## $(TAG) - $(shell date +%Y-%m-%d)/,CHANGELOG.md) - $(call exec,[ $$(git diff CHANGELOG.md 2>/dev/null |wc -l) -eq 0 ] || git commit -m "$$(cat CHANGELOG.md |sed -n '\''/$(TAG)/,/^$$/{s/##\(.*\)/release\1\n/;p;}'\'')" CHANGELOG.md) - $(call exec,[ $$(git tag -l $(TAG) |wc -l) -eq 0 ] || git tag -d $(TAG)) - $(call exec,git tag $(TAG)) - $(call exec,[ $$(git ls-remote --tags upstream $(TAG) |wc -l) -eq 0 ] || git push upstream :refs/tags/$(TAG)) - $(call exec,git push --tags upstream $*) -endif - -# Merge tag $(TAG) into upstream/$* branch -.PHONY: tag-merge-upstream-% -tag-merge-upstream-%: myos-base update-upstream -ifneq ($(words $(TAG)),0) - $(call exec,git fetch --tags -u --prune upstream $*:$*) - $(call exec,git checkout $*) - $(call exec,git merge --ff --no-edit $(TAG)) - $(call exec,git push upstream $*) -endif diff --git a/make/monorepo/release.mk b/make/monorepo/release.mk index 6c8c1fd..fa9e209 100644 --- a/make/monorepo/release.mk +++ b/make/monorepo/release.mk @@ -2,7 +2,7 @@ # RELEASE .PHONY: release -release: release-create # Create release [version] +release: release-create ## Create release [version] .PHONY: release-check release-check: @@ -20,16 +20,16 @@ endif .PHONY: release-create release-create: release-check git-stash ## Create release [version] - $(call make,branch-create-upstream-develop BRANCH=$(RELEASE_BRANCH)) + $(call make,git-branch-create-upstream-develop BRANCH=$(RELEASE_BRANCH)) $(call make,git-unstash,,STATUS) .PHONY: release-finish release-finish: release-check git-stash ## Finish release [version] - $(call make,branch-merge-upstream-master BRANCH=$(RELEASE_BRANCH)) + $(call make,git-branch-merge-upstream-master BRANCH=$(RELEASE_BRANCH)) $(call make,update-subrepos) - $(call make,tag-create-upstream-master TAG=$(RELEASE_VERSION)) + $(call make,git-tag-create-upstream-master TAG=$(RELEASE_VERSION)) $(call make,subrepos-tag-create-master TAG=$(RELEASE_VERSION)) - $(call make,tag-merge-upstream-develop TAG=$(RELEASE_VERSION)) - $(call make,branch-delete BRANCH=$(RELEASE_BRANCH)) + $(call make,git-tag-merge-upstream-develop TAG=$(RELEASE_VERSION)) + $(call make,git-branch-delete BRANCH=$(RELEASE_BRANCH)) $(call make,subrepos-branch-delete BRANCH=$(RELEASE_BRANCH)) $(call make,git-unstash,,STATUS) diff --git a/make/monorepo/update.mk b/make/monorepo/update.mk index d486f7f..1acc09d 100644 --- a/make/monorepo/update.mk +++ b/make/monorepo/update.mk @@ -12,7 +12,7 @@ endif update-$(PARAMETERS): $(PARAMETERS) $(PARAMETERS): SSH_PUBLIC_HOST_KEYS := $(PARAMETERS_REMOTE_HOST) $(SSH_BASTION_HOSTNAME) $(SSH_REMOTE_HOSTS) -$(PARAMETERS): MAKE_VARS += SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PRIVATE_IP_RANGE SSH_PUBLIC_HOST_KEYS +$(PARAMETERS): MAKE_ENV_VARS += SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PRIVATE_IP_RANGE SSH_PUBLIC_HOST_KEYS $(PARAMETERS): myos-base $(call exec,[ -d $(PARAMETERS) ] && cd $(PARAMETERS) && git pull --quiet || git clone --quiet $(APP_PARAMETERS_REPOSITORY)) diff --git a/openstack/def.openstack.mk b/openstack/def.openstack.mk index d671ad1..0b6e950 100644 --- a/openstack/def.openstack.mk +++ b/openstack/def.openstack.mk @@ -1,3 +1,4 @@ +CMDS += openstack ENV_VARS += OS_AUTH_URL OS_TENANT_ID OS_TENANT_NAME OS_USERNAME OS_PASSWORD OS_REGION_NAME OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME ifeq ($(DEBUG), true) OPENSTACK_ARGS += --debug diff --git a/stack/elastic.mk b/stack/elastic.mk index 148e5f8..84a8847 100644 --- a/stack/elastic.mk +++ b/stack/elastic.mk @@ -1,6 +1,10 @@ +ELASTICSEARCH_HOST ?= elasticsearch +ELASTICSEARCH_PORT ?= 9200 +ELASTICSEARCH_PROTOCOL ?= http +ENV_VARS += ELASTICSEARCH_HOST ELASTICSEARCH_PASSWORD ELASTICSEARCH_PORT ELASTICSEARCH_PROTOCOL ELASTICSEARCH_USERNAME + elastic ?= elastic/curator elastic/elasticsearch elastic/kibana alpine/sysctl .PHONY: elasticsearch-delete-% elasticsearch-delete-%: docker ps |awk '$$NF ~ /myos_elasticsearch/' |sed 's/^.*:\([0-9]*\)->9200\/tcp.*$$/\1/' |while read port; do echo -e "DELETE /$* HTTP/1.0\n\n" |nc localhost $$port; done -