diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ac0c2d..2fb4f18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,4 @@ ## v1.0-alpha -First public release. Code as Doc. +First public release. Code is doc. diff --git a/Makefile b/Makefile index bdbd76c..3875e52 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,6 @@ app-build: base install-build-config $(foreach service,$(or $(SERVICE),$(SERVICES)),$(call make,app-build-$(service))) $(call make,docker-commit) -app-install: base node +app-install: ansible-run base node app-start: ssh-add diff --git a/ansible/ansible.mk b/ansible/ansible.mk index 257f355..1a10c55 100644 --- a/ansible/ansible.mk +++ b/ansible/ansible.mk @@ -20,9 +20,9 @@ ansible-pull@%: ssh-get-PrivateIpAddress-$(SERVER_NAME) # target ansible-run: Fire ssh-add ansible-run-localhost .PHONY: ansible-run -ansible-run: ssh-add ansible-run-localhost +ansible-run: ansible-run-localhost # target ansible-run-%: Fire docker-build-ansible, Call ansible-playbook ANSIBLE_PLAYBOOK .PHONY: ansible-run-% -ansible-run-%: $(if $(DOCKER_RUN),docker-build-ansible) +ansible-run-%: $(if $(DOCKER_RUN),docker-build-ansible,install-ansible) $(call ansible-playbook,$(if $(ANSIBLE_TAGS),--tags $(ANSIBLE_TAGS)) $(if $(ANSIBLE_EXTRA_VARS),--extra-vars '$(patsubst target=localhost,target=$*,$(ANSIBLE_EXTRA_VARS))') $(if $(findstring true,$(DRYRUN)),--check) $(if $(ANSIBLE_INVENTORY),--inventory $(ANSIBLE_INVENTORY)) $(ANSIBLE_PLAYBOOK)) diff --git a/ansible/def.ansible.mk b/ansible/def.ansible.mk index 6b935a1..95f3675 100644 --- a/ansible/def.ansible.mk +++ b/ansible/def.ansible.mk @@ -1,3 +1,4 @@ +ANSIBLE_ARGS ?= $(if $(filter-out 0,$(UID)),$(if $(shell sudo -l 2>/dev/null |grep 'NOPASSWD: ALL'),,-K))$(if $(DOCKER_RUN),$(if $(shell ssh-add -l >/dev/null 2>&1 || echo false), -k)) ANSIBLE_AWS_ACCESS_KEY_ID ?= $(AWS_ACCESS_KEY_ID) ANSIBLE_AWS_DEFAULT_OUTPUT ?= $(AWS_DEFAULT_OUTPUT) ANSIBLE_AWS_DEFAULT_REGION ?= $(AWS_DEFAULT_REGION) @@ -9,19 +10,25 @@ ANSIBLE_DISKS_NFS_PATH ?= $(NFS_PATH) ANSIBLE_DOCKER_IMAGE_TAG ?= $(DOCKER_IMAGE_TAG) ANSIBLE_DOCKER_REGISTRY ?= $(DOCKER_REGISTRY) ANSIBLE_EXTRA_VARS ?= target=localhost -ANSIBLE_GIT_DIRECTORY ?= /src/$(subst git@,,$(subst ssh://,,$(GIT_REPOSITORY))) +ANSIBLE_GIT_DIRECTORY ?= /src/$(subst $(space),/,$(strip $(call reverse,$(subst ., ,$(APP_REPOSITORY_HOST)))))/$(APP_REPOSITORY_PATH) ANSIBLE_GIT_KEY_FILE ?= $(if $(ANSIBLE_SSH_PRIVATE_KEYS),~$(ANSIBLE_USERNAME)/.ssh/$(notdir $(firstword $(ANSIBLE_SSH_PRIVATE_KEYS)))) ANSIBLE_GIT_REPOSITORY ?= $(GIT_REPOSITORY) ANSIBLE_GIT_VERSION ?= $(BRANCH) ANSIBLE_INVENTORY ?= ansible/inventories ANSIBLE_PLAYBOOK ?= ansible/playbook.yml -ANSIBLE_SSH_PRIVATE_KEYS ?= $(SSH_PRIVATE_KEYS) +ANSIBLE_SSH_AUTHORIZED_KEYS ?= $(strip $(SSH_AUTHORIZED_KEYS)) +ANSIBLE_SSH_BASTION_HOSTNAME ?= $(firstword $(SSH_BASTION_HOSTNAME)) +ANSIBLE_SSH_BASTION_USERNAME ?= $(firstword $(SSH_BASTION_USERNAME)) +ANSIBLE_SSH_PRIVATE_IP_RANGE ?= $(strip $(SSH_PRIVATE_IP_RANGE)) +ANSIBLE_SSH_PRIVATE_KEYS ?= $(strip $(SSH_PRIVATE_KEYS)) +ANSIBLE_SSH_PUBLIC_HOSTS ?= $(strip $(SSH_PUBLIC_HOSTS)) +ANSIBLE_SSH_USERNAME ?= $(firstword $(SSH_USER)) ANSIBLE_SERVER_NAME ?= $(SERVER_NAME) -ANSIBLE_USERNAME ?= root +ANSIBLE_USERNAME ?= $(USER) ANSIBLE_VERBOSE ?= $(if $(DEBUG),-vvvv,$(if $(VERBOSE),-v)) CMDS += ansible ansible-playbook DOCKER_RUN_OPTIONS += --add-host=host.docker.internal:$(DOCKER_INTERNAL_DOCKER_HOST) -ENV_VARS += ANSIBLE_AWS_ACCESS_KEY_ID ANSIBLE_AWS_DEFAULT_OUTPUT ANSIBLE_AWS_DEFAULT_REGION ANSIBLE_AWS_SECRET_ACCESS_KEY ANSIBLE_CONFIG ANSIBLE_DISKS_NFS_DISK ANSIBLE_DISKS_NFS_OPTIONS ANSIBLE_DISKS_NFS_PATH ANSIBLE_DOCKER_IMAGE_TAG ANSIBLE_DOCKER_REGISTRY ANSIBLE_EXTRA_VARS ANSIBLE_GIT_DIRECTORY ANSIBLE_GIT_KEY_FILE ANSIBLE_GIT_REPOSITORY ANSIBLE_GIT_VERSION ANSIBLE_INVENTORY ANSIBLE_PLAYBOOK ANSIBLE_SSH_PRIVATE_KEYS ANSIBLE_USERNAME ANSIBLE_VERBOSE +ENV_VARS += ANSIBLE_AWS_ACCESS_KEY_ID ANSIBLE_AWS_DEFAULT_OUTPUT ANSIBLE_AWS_DEFAULT_REGION ANSIBLE_AWS_SECRET_ACCESS_KEY ANSIBLE_CONFIG ANSIBLE_DISKS_NFS_DISK ANSIBLE_DISKS_NFS_OPTIONS ANSIBLE_DISKS_NFS_PATH ANSIBLE_DOCKER_IMAGE_TAG ANSIBLE_DOCKER_REGISTRY ANSIBLE_EXTRA_VARS ANSIBLE_GIT_DIRECTORY ANSIBLE_GIT_KEY_FILE ANSIBLE_GIT_REPOSITORY ANSIBLE_GIT_VERSION ANSIBLE_INVENTORY ANSIBLE_PLAYBOOK ANSIBLE_SSH_AUTHORIZED_KEYS ANSIBLE_SSH_BASTION_HOSTNAME ANSIBLE_SSH_BASTION_USERNAME ANSIBLE_SSH_PRIVATE_IP_RANGE ANSIBLE_SSH_PRIVATE_KEYS ANSIBLE_SSH_PUBLIC_HOSTS ANSIBLE_SSH_USERNAME ANSIBLE_USERNAME ANSIBLE_VERBOSE # function ansible: Call run ansible ANSIBLE_ARGS with arg 1 define ansible @@ -37,5 +44,5 @@ endef ## TODO: run ansible in docker and target localhost outside docker when DOCKER=true define ansible-pull $(call INFO,ansible-pull,$(1)) - $(RUN) $(call env-run,ansible-pull $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) + $(call env-run,$(RUN) ansible-pull $(ANSIBLE_ARGS) $(ANSIBLE_VERBOSE) $(1)) endef diff --git a/ansible/install.mk b/ansible/install.mk new file mode 100644 index 0000000..26ad8b3 --- /dev/null +++ b/ansible/install.mk @@ -0,0 +1,8 @@ +## +# INSTALL + +# target install-ansible; Install ansible on local host +.PHONY: install-ansible +install-ansible: + $(if $(shell type -p ansible),,$(RUN) $(INSTALL) ansible) + diff --git a/ansible/inventories/host_vars/default b/ansible/inventories/host_vars/default index a58b287..b0816ce 100644 --- a/ansible/inventories/host_vars/default +++ b/ansible/inventories/host_vars/default @@ -16,21 +16,12 @@ hosts_git_repositories: - { "repo": "{{ lookup('env','ANSIBLE_GIT_REPOSITORY') }}", "dest": "{{ lookup('env','ANSIBLE_GIT_DIRECTORY') }}", "key_file": "{{ lookup('env','ANSIBLE_GIT_KEY_FILE') or '~/.ssh/id_rsa' }}", "version": "{{ lookup('env','ANSIBLE_GIT_VERSION') }}" } hosts_packages: - { "name": "ansible", "state": "present" } -- { "name": "coreutils", "state": "present" } -- { "name": "curl", "state": "present" } -- { "name": "git", "state": "present" } -- { "name": "groff", "state": "present" } -- { "name": "htop", "state": "present" } -- { "name": "less", "state": "present" } -- { "name": "lsof", "state": "present" } -- { "name": "make", "state": "present" } -- { "name": "openssh-client", "state": "present" } -- { "name": "util-linux", "state": "present" } -- { "name": "vim", "state": "present" } -- { "name": "zsh", "state": "present" } -hosts_ssh_private_keys: "{{ lookup('env','ANSIBLE_SSH_PRIVATE_KEYS').split(' ').default([]) }}" -hosts_ssh_users: -- aya +hosts_rc_functions: + - 10_prompt_set + - 10_ps1_set + - 30_pfetch + - 30_screen_attach + - 40_ssh_add hosts_user_env: - ANSIBLE_AWS_ACCESS_KEY_ID - ANSIBLE_AWS_SECRET_ACCESS_KEY diff --git a/ansible/inventories/host_vars/localhost b/ansible/inventories/host_vars/localhost index b6f4970..88b624d 100644 --- a/ansible/inventories/host_vars/localhost +++ b/ansible/inventories/host_vars/localhost @@ -1,25 +1,23 @@ --- # file: inventories/host_vars/localhost -aws_access_key_id: "{{ lookup('env','ANSIBLE_AWS_ACCESS_KEY_ID') }}" -aws_output_format: "{{ lookup('env','ANSIBLE_AWS_DEFAULT_OUTPUT') or 'json' }}" -aws_region: "{{ lookup('env','ANSIBLE_AWS_DEFAULT_REGION') or 'eu-west-1' }}" -aws_secret_access_key: "{{ lookup('env','ANSIBLE_AWS_SECRET_ACCESS_KEY') }}" -disks_additional_disks: -- disk: /dev/xvdb - disable_periodic_fsck: true - fstype: ext4 - mount_options: defaults - mount: /var/lib/docker - service: docker -- disk: "{{ lookup('env','ANSIBLE_DISKS_NFS_DISK') }}" - fstype: nfs - mount_options: "{{ lookup('env','ANSIBLE_DISKS_NFS_OPTIONS') }}" - mount: "{{ lookup('env','ANSIBLE_DISKS_NFS_PATH') }}" -disks_additional_services: -- rpc.statd docker_image_tag: "{{ lookup('env','ANSIBLE_DOCKER_IMAGE_TAG') or 'latest' }}" docker_registry: "{{ lookup('env','ANSIBLE_DOCKER_REGISTRY') }}" hosts_enable_local: true hosts_enable_rc: true hosts_enable_zram: true +hosts_git_repositories: +- { "repo": "{{ lookup('env','ANSIBLE_GIT_REPOSITORY') }}", "dest": "{{ lookup('env','ANSIBLE_GIT_DIRECTORY') }}", "key_file": "{{ lookup('env','ANSIBLE_GIT_KEY_FILE') or '~/.ssh/id_rsa' }}", "version": "{{ lookup('env','ANSIBLE_GIT_VERSION') }}" } +hosts_packages: +- { "name": "ansible", "state": "present" } +hosts_user_env: +- ANSIBLE_CONFIG +- ANSIBLE_DOCKER_IMAGE_TAG +- ANSIBLE_DOCKER_REGISTRY +- ANSIBLE_EXTRA_VARS +- ANSIBLE_GIT_DIRECTORY +- ANSIBLE_GIT_KEY_FILE +- ANSIBLE_GIT_REPOSITORY +- ANSIBLE_INVENTORY +- ANSIBLE_PLAYBOOK +- ENV diff --git a/ansible/inventories/host_vars/myos b/ansible/inventories/host_vars/myos new file mode 100644 index 0000000..2f1f63f --- /dev/null +++ b/ansible/inventories/host_vars/myos @@ -0,0 +1,49 @@ +--- +# file: inventories/host_vars/myos + +aws_access_key_id: "{{ lookup('env','ANSIBLE_AWS_ACCESS_KEY_ID') }}" +aws_output_format: "{{ lookup('env','ANSIBLE_AWS_DEFAULT_OUTPUT') or 'json' }}" +aws_region: "{{ lookup('env','ANSIBLE_AWS_DEFAULT_REGION') or 'eu-west-1' }}" +aws_secret_access_key: "{{ lookup('env','ANSIBLE_AWS_SECRET_ACCESS_KEY') }}" +disks_additional_disks: +- disk: /dev/xvdb + disable_periodic_fsck: true + fstype: ext4 + mount_options: defaults + mount: /var/lib/docker + service: docker +- disk: "{{ lookup('env','ANSIBLE_DISKS_NFS_DISK') }}" + fstype: nfs + mount_options: "{{ lookup('env','ANSIBLE_DISKS_NFS_OPTIONS') }}" + mount: "{{ lookup('env','ANSIBLE_DISKS_NFS_PATH') }}" +disks_additional_services: +- rpc.statd +docker_image_tag: "{{ lookup('env','ANSIBLE_DOCKER_IMAGE_TAG') or 'latest' }}" +docker_registry: "{{ lookup('env','ANSIBLE_DOCKER_REGISTRY') }}" +hosts_enable_local: true +hosts_enable_rc: true +hosts_enable_zram: true +hosts_packages: +- { "name": "ansible", "state": "present" } +hosts_rc_functions: + - 10_prompt_set + - 10_ps1_set + - 30_pfetch + - 30_screen_attach + - 40_ssh_add +hosts_user_env: +- ANSIBLE_AWS_ACCESS_KEY_ID +- ANSIBLE_AWS_SECRET_ACCESS_KEY +- ANSIBLE_CONFIG +- ANSIBLE_DISKS_NFS_DISK +- ANSIBLE_DISKS_NFS_OPTIONS +- ANSIBLE_DISKS_NFS_PATH +- ANSIBLE_DOCKER_IMAGE_TAG +- ANSIBLE_DOCKER_REGISTRY +- ANSIBLE_EXTRA_VARS +- ANSIBLE_GIT_DIRECTORY +- ANSIBLE_GIT_KEY_FILE +- ANSIBLE_GIT_REPOSITORY +- ANSIBLE_INVENTORY +- ANSIBLE_PLAYBOOK +- ENV diff --git a/ansible/playbooks/aws-cli.yml b/ansible/playbooks/aws-cli.yml index df5c217..c7e796c 100644 --- a/ansible/playbooks/aws-cli.yml +++ b/ansible/playbooks/aws-cli.yml @@ -1,6 +1,6 @@ --- # file: playbooks/aws-cli.yml -- hosts: '{{ target | default("all") }}' +- hosts: '{{ target | default("aws") }}' roles: - aws-cli diff --git a/ansible/playbooks/disks.yml b/ansible/playbooks/disks.yml index 7ea9505..bd1fa1e 100644 --- a/ansible/playbooks/disks.yml +++ b/ansible/playbooks/disks.yml @@ -1,6 +1,6 @@ --- # file: playbooks/disks.yml -- hosts: '{{ target | default("all") }}' +- hosts: '{{ target | default("disks") }}' roles: - disks diff --git a/ansible/roles/aws-cli/tasks/main.yml b/ansible/roles/aws-cli/tasks/main.yml index 44b519c..1924c5a 100644 --- a/ansible/roles/aws-cli/tasks/main.yml +++ b/ansible/roles/aws-cli/tasks/main.yml @@ -101,16 +101,17 @@ tags: 'aws' when: ec2_tags.tags is defined -- name: aws - prune docker objects (including non-dangling images) - docker_prune: - containers: yes - images: yes - images_filters: - dangling: false - networks: yes - volumes: yes - builder_cache: yes - tags: 'aws' +# ansible v2.8 +# - name: aws - prune docker objects +# docker_prune: +# containers: yes +# images: yes +# images_filters: +# dangling: false +# networks: yes +# volumes: yes +# builder_cache: yes +# tags: 'aws' - name: aws - launch docker containers docker_container: diff --git a/ansible/roles/disks/tasks/main.yml b/ansible/roles/disks/tasks/main.yml index 8ee0133..63524d8 100644 --- a/ansible/roles/disks/tasks/main.yml +++ b/ansible/roles/disks/tasks/main.yml @@ -3,23 +3,27 @@ name=py3-pip state=present when: ansible_os_family|lower == "alpine" + become: yes - name: 'Install Python PIP' package: > name=python-pip state=present when: ansible_os_family|lower != "alpine" + become: yes - name: 'Install python-pathlib' pip: > name=pathlib state=present + become: yes - name: "Discover NVMe EBS" disks_ebs_config: config: "{{ disks_additional_disks }}" register: __disks_ebs_config when: disks_discover_aws_nvme_ebs | default(True) | bool + become: yes - set_fact: disks_additional_disks: "{{ disks_additional_disks|defaut([]) + __disks_ebs_config['ansible_facts']['config'] }}" @@ -32,6 +36,7 @@ use: '{{ disks_package_use }}' when: disks_additional_disks tags: ['disks', 'pkgs'] + become: yes - name: "Install additional fs progs" package: @@ -40,6 +45,7 @@ with_items: "{{ disks_additional_packages|default([]) }}" when: disks_additional_packages is defined tags: ['disks', 'pkgs'] + become: yes - name: disks - start additional services service: @@ -48,6 +54,7 @@ state: started with_items: "{{ disks_additional_services|default([]) }}" tags: ['disks', 'pkgs'] + become: yes - name: "Get disk alignment for disks" shell: | @@ -85,6 +92,7 @@ executable: '/bin/bash' with_items: '{{ disks_additional_disks }}' tags: ['disks'] + become: yes - name: "Create filesystem on the first partition" filesystem: @@ -97,6 +105,7 @@ - '{{ disks_stat.results }}' when: item.1.stat.exists tags: ['disks'] + become: yes - name: "Disable periodic fsck and reserved space on ext3 or ext4 formatted disks" environment: @@ -107,6 +116,7 @@ - '{{ disks_stat.results }}' when: "disks_additional_disks and ( item.0.fstype == 'ext4' or item.0.fstype == 'ext3' ) and item.0.disable_periodic_fsck|default(false)|bool and item.1.stat.exists" tags: ['disks'] + become: yes - name: "Ensure the mount directory exists" file: @@ -114,6 +124,7 @@ state: directory with_items: '{{ disks_additional_disks }}' tags: ['disks'] + become: yes - name: "Get UUID for partition" environment: @@ -145,6 +156,7 @@ register: disks_additional_disks_handler_notify notify: - restart services + become: yes - name: "Mount additional disks - nfs" mount: @@ -159,6 +171,7 @@ register: disks_additional_disks_nfs_handler_notify notify: - restart services - nfs + become: yes - name: "Ensure the permissions are set correctly" file: @@ -168,6 +181,7 @@ state: directory with_items: '{{ disks_additional_disks }}' when: item.user is defined or item.group is defined - tags: ['disk'] + tags: ['disks'] + become: yes - meta: flush_handlers diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml index e24cd12..7e0d516 100644 --- a/ansible/roles/docker/defaults/main.yml +++ b/ansible/roles/docker/defaults/main.yml @@ -4,9 +4,6 @@ # minimum kernel version docker_check_kernel: '3.10' -# architecture -docker_check_machine: 'x86_64' - # The docker package name docker_package: docker diff --git a/ansible/roles/docker/tasks/check.yml b/ansible/roles/docker/tasks/check.yml index 471cdff..fb33a3a 100644 --- a/ansible/roles/docker/tasks/check.yml +++ b/ansible/roles/docker/tasks/check.yml @@ -7,10 +7,3 @@ docker requires a minimum kernel version of {{docker_check_kernel}} on {{ansible_distribution}} {{ansible_distribution_version}} when: ansible_kernel is version(docker_check_kernel, "<") - -- name: check - machine architecture - fail: - msg: > - docker requires a {{docker_check_machine}} version - of {{ansible_distribution}} {{ansible_distribution_version}} - when: ansible_machine != docker_check_machine diff --git a/ansible/roles/docker/tasks/package.yml b/ansible/roles/docker/tasks/package.yml index 88c2eca..364f501 100644 --- a/ansible/roles/docker/tasks/package.yml +++ b/ansible/roles/docker/tasks/package.yml @@ -9,11 +9,17 @@ - name: package - add docker GPG key apt_key: url=https://download.docker.com/linux/debian/gpg when: ansible_os_family|lower == "debian" + become: yes + +- name: package - define arch + set_fact: + docker_apt_arch: "{% if ansible_machine == 'aarch64' %}arm64{% endif %}{% if ansible_machine == 'x86_64' %}amd64{% endif %}" - name: add docker APT repository apt_repository: - repo: deb [arch=amd64] https://download.docker.com/linux/{{ansible_distribution|lower}} {{ansible_distribution_release}} stable + repo: deb [arch={{docker_apt_arch}}] https://download.docker.com/linux/{{ansible_distribution|lower}} {{ansible_distribution_release}} stable when: ansible_os_family|lower == "debian" + become: yes - name: package - add docker package package: name="{{docker_package}}" state=present diff --git a/ansible/roles/hosts/README.md b/ansible/roles/hosts/README.md index 64c7ed3..d002ebd 100644 --- a/ansible/roles/hosts/README.md +++ b/ansible/roles/hosts/README.md @@ -1,21 +1,21 @@ # Ansible role to customize servers -An ansible role to customize your servers after a fresh install +An ansible role to customize servers after a fresh install ## Role Variables -* `hosts_ssh_users` - A list of github usernames. We will fetch ssh keys from their github account and add it to the authorized_keys of the ansible user. +* `hosts_enable_cloudinit` - Install and configure cloud-init ``` yaml -# a list of github usernames to get public keys -hosts_ssh_users: [] +# enable cloud-init +hosts_enable_cloudinit: false ``` -* `hosts_enable_zram` - Activate zram swap devices. This option allows to create virtual swap devices compressed in RAM. It can increase hosts performances, specially on hosts without physical swap. +* `hosts_enable_local` - Run ansible pull at boot ``` yaml -# Activate zram swap devices -hosts_enable_zram: false +# enable rc.local script +hosts_enable_local: false ``` * `hosts_enable_rc` - Run user specific functions on ssh connection. This allow a user to customize his session when connecting to a server, like attaching automaticaly a screen session for example. @@ -25,37 +25,19 @@ hosts_enable_zram: false hosts_enable_rc: false ``` -* `hosts_rc_functions` - List of user specific functions to run on ssh connection. Here you can add any function to be called when you connect to the host. Default functions are available in the /etc/profile.d/rc_functions.sh file. +* `hosts_enable_zram` - Activate zram swap devices. This option allows to create virtual swap devices compressed in RAM. It can increase hosts performances, specially on hosts without physical swap. ``` yaml -# list of rc functions to call at user connection -hosts_rc_functions: - # customize PS1 variable - - 01_custom_ps1 - # customize PROMPT variable - # - 02_custom_prompt - # launch a ssh agent and load all private keys located in ~/.ssh - # - 03_ssh_agent - # create and/or attach a tmux session - # - 04_attach_tmux - # create and/or attach a screen session - - 05_attach_screen +# Activate zram swap devices +hosts_enable_zram: false ``` -* `hosts_rc_cleanup` - List of rc functions you do not want to run anymore. If you had previously activated a rc function in `hosts_rc_functions`, you can add it to `hosts_rc_cleanup` to disable it. +* `hosts_git_repositories` - Clone git repositories. ``` yaml -# list of rc functions to cleanup (remove files) -# hosts_rc_cleanup: - # - 03_ssh_agent - # - 04_attach_tmux -``` - -* `hosts_etc_bashrc` - The location of the /etc/bashrc file on the current distro - -``` yaml -# location of /etc/bashrc -hosts_etc_bashrc: /etc/bashrc +# git repositories to clone +hosts_git_repositories: +- { "repo": "https://github.com/aya/myos", "dest": "/src/com/github/aya/myos", "key_file": "~/.ssh/id_rsa", "version": "master" } ``` * `hosts_packages` - A list of packages to install on your servers. This list should be overrided for a specific distro. @@ -70,11 +52,101 @@ hosts_packages: [] ``` yaml # packages common to all distributions hosts_packages_common: - - { "name": "bash", "state": "present" } - - { "name": "ca-certificates", "state": "present" } - - { "name": "rsync", "state": "present" } - - { "name": "screen", "state": "present" } - - { "name": "tzdata", "state": "present" } +- { "name": "bash", "state": "present" } +``` + +* `hosts_packages_distro` - A list of packages to install on your servers. This list is specific to your distro. + +``` yaml +# packages specific to a distribution +hosts_packages_distro: +- { "name": "vim-nox", "state": "present" } +``` + +* `hosts_rc_functions` - List of user specific functions to run on ssh connection. Here you can add any function to be called when you connect to the host. Default functions are available in the /etc/profile.d/rc_functions.sh file. + +``` yaml +# list of rc functions to call at user connection +hosts_rc_functions: +# load shell functions +- 00_source +# customize PROMPT variable +- 10_prompt_set +# customize PS1 variable +- 10_ps1_set +# create and/or attach a tmux session +- 20_tmux_attach +# display host infos +- 30_pfetch +# create and/or attach a screen session +- 30_screen_attach +# launch ssh agent and load private keys in ~/.ssh +- 40_ssh_add +``` + +* `hosts_rc_cleanup` - List of rc functions you do not want to run anymore. If you had previously activated a rc function in `hosts_rc_functions`, you can add it to `hosts_rc_cleanup` to disable it. + +``` yaml +# list of rc functions to cleanup (remove files) +hosts_rc_cleanup: +- 01_custom_ps1 +- 02_custom_prompt +- 03_ssh_agent +- 04_attach_tmux +- 05_attach_screen +``` + +* `hosts_ssh_authorized_keys` - A list of urls. Fetch ssh public keys from urls and add them to file ~/.ssh/authorized_keys of the ansible user. + +``` yaml +# a list of urls to get ssh public keys +hosts_ssh_authorized_keys: +- https://github.com/aya.keys +``` + +* `hosts_ssh_bastion_hostname` - Hostname of ssh bastion. Needed to add myos-bastion to file ~/.ssh/myos/config of the ansible user. + +``` yaml +# hostname of myos-bastion to add in ~/.ssh/myos/config +hosts_ssh_bastion_hostname: 8.4.2.1 +``` + +* `hosts_ssh_bastion_username` - Username of ssh bastion. Needed to add myos-bastion to file ~/.ssh/myos/config of the ansible user. + +``` yaml +# hostname of myos-bastion to add in ~/.ssh/myos/config +hosts_ssh_bastion_username: root +``` + +* `hosts_ssh_private_ip_range` - Ip range to pass through ssh bastion. + +``` yaml +# ip range proxyfied through myos-bastion to add in ~/.ssh/myos/config +hosts_ssh_private_ip_range: 10.* 192.168.42.* +``` + +* `hosts_ssh_private_keys` - A list of ssh private keys to copy. Default to ~/.ssh/id_rsa + +``` yaml +# a list of urls to get ssh public keys +hosts_ssh_private_keys: +- ~/.ssh/id_rsa +``` + +* `hosts_ssh_public_hosts` - A list of host names to get ssh fingerprint + +``` yaml +# a list of public hosts to add to ~/.ssh/known_hosts +hosts_ssh_public_hosts: +- github.com +- gitlab.com +``` + +* `hosts_ssh_username` - ssh user used to ssh on remote hosts + +``` yaml +# ssh username to ssh on remote hosts +hosts_ssh_username: root ``` ## Example @@ -85,22 +157,6 @@ To launch this role on your `hosts` servers, run the default playbook. $ ansible-playbook playbook.yml ``` -It will install the following packages : bash, ca-certificates, rsync, screen, tzdata and vim (plus libselinux-python on redhat). - -## Common configurations - -This example configuration will add the [ssh keys from aya's github user](https://github.com/aya.keys) to your remote ~/.ssh/authorized_keys. -It will create a ~/.rc.d and touch 01_custom_ps1 and 02_attach_screen files into this directory, resulting in a customized PS1 and automaticaly attaching a screen on (re)connection on the remote server. - -``` yaml -hosts_ssh_users: - - aya -hosts_enable_rc: true -hosts_rc_functions: - - 01_custom_ps1 - - 02_attach_screen -``` - ## Tests To test this role on your `hosts` servers, run the tests/playbook.yml playbook. diff --git a/ansible/roles/hosts/defaults/main.yml b/ansible/roles/hosts/defaults/main.yml index fcb2d19..beb122f 100644 --- a/ansible/roles/hosts/defaults/main.yml +++ b/ansible/roles/hosts/defaults/main.yml @@ -1,72 +1,7 @@ --- # file: defaults/main.yml -# enable cloud-init -hosts_enable_cloudinit: false - -# enable rc.local script -hosts_enable_local: false - -# run user specific rc functions on ssh connection -hosts_enable_rc: false - -# Activate zram swap devices on host -hosts_enable_zram: false - -# git repositories to clone -hosts_git_repositories: [] -# - { "repo": "ssh://git@github.com/aya/infra", "dest": "/src" } - -# list of rc functions to call at user connection -hosts_rc_functions: - # customize PS1 variable - - 01_custom_ps1 - # customize PROMPT variable - - 02_custom_prompt - # launch a ssh agent and load all private keys located in ~/.ssh - - 03_ssh_agent - # create and/or attach a tmux session - # - 04_attach_tmux - # create and/or attach a screen session - - 05_attach_screen - # display system information - - 06_pfetch - -# list of rc functions to cleanup (remove files) -# hosts_rc_cleanup: -# - 03_ssh_agent -# - 04_attach_tmux - -# packages to install -hosts_packages: [] - -# packages specific to a distribution -hosts_packages_distro: [] - -# packages common to all distributions -hosts_packages_common: - - { "name": "bash", "state": "present" } - - { "name": "ca-certificates", "state": "present" } - - { "name": "rsync", "state": "present" } - - { "name": "screen", "state": "present" } - - { "name": "tzdata", "state": "present" } - -# a list of SSH private keys to copy -hosts_ssh_private_keys: [] -# - ~/.ssh/id_rsa - -# a list of public hosts keys to add to known_hosts -hosts_ssh_public_hosts_keys: - - { "name": "github.com", "key": "files/etc/ssh/github.com.pub" } - -# a list of github usernames to get public keys -hosts_ssh_users: [] -# - aya - -# a list of environment variables to write to user ~/.env -hosts_user_env: [] -# - SHELL - +# cloud-init config hosts_cloudinit_config: users: - default @@ -140,3 +75,72 @@ hosts_cloudinit_config: templates_dir: /etc/cloud/templates/ ssh_svcname: sshd +# enable cloud-init +hosts_enable_cloudinit: false + +# enable rc.local script +hosts_enable_local: false + +# run user specific rc functions on ssh connection +hosts_enable_rc: false + +# Activate zram swap devices on host +hosts_enable_zram: false + +# git repositories to clone +hosts_git_repositories: [] + +# packages to install +hosts_packages: [] + +# packages common to all distributions +hosts_packages_common: + - { "name": "bash", "state": "present" } + - { "name": "ca-certificates", "state": "present" } + - { "name": "curl", "state": "present" } + - { "name": "git", "state": "present" } + - { "name": "htop", "state": "present" } + - { "name": "less", "state": "present" } + - { "name": "lsof", "state": "present" } + - { "name": "make", "state": "present" } + - { "name": "rsync", "state": "present" } + - { "name": "screen", "state": "present" } + - { "name": "tmux", "state": "present" } + - { "name": "tzdata", "state": "present" } + +# packages specific to a distribution +hosts_packages_distro: [] + +# list of rc functions to cleanup (remove files) +hosts_rc_cleanup: [] + +# list of rc functions to call at user connection +hosts_rc_functions: + - 10_prompt_set + - 10_ps1_set + - 30_pfetch + - 40_ssh_add + +# a list of urls to get public keys to add to ~/.ssh/authorized_keys +hosts_ssh_authorized_keys: "{{ lookup('env','ANSIBLE_SSH_AUTHORIZED_KEYS').split(' ') }}" + +# hostname of myos-bastion to add in ~/.ssh/myos/config +hosts_ssh_bastion_hostname: "{{ lookup('env','ANSIBLE_SSH_BASTION_HOSTNAME') }}" + +# username of myos-bastion to add in ~/.ssh/myos/config +hosts_ssh_bastion_username: "{{ lookup('env','ANSIBLE_SSH_BASTION_USERNAME') or ansible_user }}" + +# ip range proxyfied through myos-bastion to add in ~/.ssh/myos/config +hosts_ssh_private_ip_range: "{{ lookup('env','ANSIBLE_SSH_PRIVATE_IP_RANGE') }}" + +# a list of SSH private keys to copy +hosts_ssh_private_keys: "{{ lookup('env','ANSIBLE_SSH_PRIVATE_KEYS').split(' ') }}" + +# a list of public hosts to add to known_hosts +hosts_ssh_public_hosts: "{{ lookup('env','ANSIBLE_SSH_PUBLIC_HOSTS').split(' ') }}" + +# remote ssh user +hosts_ssh_username: "{{ lookup('env','ANSIBLE_SSH_USERNAME') or ansible_user }}" + +# a list of environment variables to write to user ~/.env +hosts_user_env: [] diff --git a/ansible/roles/hosts/files/.bash_profile b/ansible/roles/hosts/files/.bash_profile new file mode 100644 index 0000000..d7a82fd --- /dev/null +++ b/ansible/roles/hosts/files/.bash_profile @@ -0,0 +1,18 @@ +# ~/.bash_profile: executed by the command interpreter for bash login shell. + +[ -f ~/.profile ] && source ~/.profile + +# bash-completion +if ! shopt -oq posix && [ -z "${BASH_COMPLETION_VERSINFO-}" ]; then + if [ ${BASH_VERSINFO[0]} -gt 4 ] \ + || [ ${BASH_VERSINFO[0]} -eq 4 -a ${BASH_VERSINFO[1]} -ge 1 ]; then + shopt -q progcomp && for file in \ + /{*/local,usr}/share/bash-completion/bash_completion \ + /etc/bash_completion; do + [ -r "$file" ] && . "$file" + done + fi + if [ -f "${XDG_CONFIG_HOME:-$HOME/.config}/bash_completion" ]; then + . "${XDG_CONFIG_HOME:-$HOME/.config}/bash_completion" + fi +fi diff --git a/ansible/roles/hosts/files/.dircolors_aliases b/ansible/roles/hosts/files/.dircolors_aliases new file mode 100644 index 0000000..e578625 --- /dev/null +++ b/ansible/roles/hosts/files/.dircolors_aliases @@ -0,0 +1,7 @@ +test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" +alias ls='ls --color=auto' +alias dir='dir --color=auto' +alias vdir='vdir --color=auto' +alias grep='grep --color=auto' +alias fgrep='fgrep --color=auto' +alias egrep='egrep --color=auto' diff --git a/ansible/roles/hosts/files/.docker_aliases b/ansible/roles/hosts/files/.docker_aliases new file mode 100644 index 0000000..b391747 --- /dev/null +++ b/ansible/roles/hosts/files/.docker_aliases @@ -0,0 +1 @@ +alias ctop='docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock:ro quay.io/vektorlab/ctop:latest' diff --git a/ansible/roles/hosts/files/.profile b/ansible/roles/hosts/files/.profile new file mode 100644 index 0000000..3ecd639 --- /dev/null +++ b/ansible/roles/hosts/files/.profile @@ -0,0 +1,38 @@ +# ~/.profile: executed by the command interpreter for login shells. +# set -x +umask 027 + +# source ~/.*aliases and ~/.*functions files +for source in aliases functions; do + for file in "$HOME"/.*"$source"; do + [ -f "$file" ] || continue + # remove $HOME/. prefix from file + file="${file#${HOME}/.}" + # remove _$source suffix from $file + command="${file%_$source}" + # source file if command exists, ie ~/.bash_aliases + command -v "$command" >/dev/null 2>&1 && . "${HOME}/.$file" + # remove $source suffix from $file, ie ~/.aliases + command="${file%$source}" + # source file if command empty, ie ~/.aliases + [ -z "$command" ] && . "${HOME}/.$file" + done +done + +# source ~/.*shrc +for file in "$HOME"/.*shrc; do + [ -f "$file" ] || continue + # remove $HOME/. prefix from file + file="${file#${HOME}/.}" + # source file if match current shell + [ "$(basename ${SHELL})" = "${file%rc}" ] && . "${HOME}/.$file" +done + +# set PATH to include user's bin +for path in /*/local/sbin /*/local/bin /*/local/*/bin "${HOME}"/.*/bin; do + [ -d "$path" ] || continue + case ":${PATH}:" in + *:"$path":*) ;; + *) export PATH="${path}:$PATH" ;; + esac +done diff --git a/ansible/roles/hosts/files/.resty_aliases b/ansible/roles/hosts/files/.resty_aliases new file mode 100644 index 0000000..2fb9e14 --- /dev/null +++ b/ansible/roles/hosts/files/.resty_aliases @@ -0,0 +1,8 @@ +alias DELETE='resty-delete' +alias GET='resty-get' +alias HEAD='resty-head' +alias OPTIONS='resty-options' +alias PATCH='resty-patch' +alias POST='resty-post' +alias PUT='resty-put' +alias TRACE='resty-trace' diff --git a/ansible/roles/hosts/files/.sh_functions b/ansible/roles/hosts/files/.sh_functions new file mode 100644 index 0000000..e698463 --- /dev/null +++ b/ansible/roles/hosts/files/.sh_functions @@ -0,0 +1,2 @@ +gam () { "~/.gam/gam" "$@" ; } +gyb () { "~/.gyb/gyb" "$@" ; } diff --git a/ansible/roles/hosts/files/.shrc b/ansible/roles/hosts/files/.shrc new file mode 100644 index 0000000..6ffc0a6 --- /dev/null +++ b/ansible/roles/hosts/files/.shrc @@ -0,0 +1,5 @@ +export EDITOR=vim +export GIT_PS1_SHOWUPSTREAM=auto +export GIT_PS1_SHOWDIRTYSTATE=true +export GIT_PS1_HIDE_IF_PWD_IGNORED=true +export PAGER=less diff --git a/ansible/roles/hosts/files/.vim_aliases b/ansible/roles/hosts/files/.vim_aliases new file mode 100644 index 0000000..924a904 --- /dev/null +++ b/ansible/roles/hosts/files/.vim_aliases @@ -0,0 +1 @@ +alias vi='vim' diff --git a/ansible/roles/hosts/files/etc/profile.d/rc.sh b/ansible/roles/hosts/files/etc/profile.d/rc.sh index 6851226..b37a503 100644 --- a/ansible/roles/hosts/files/etc/profile.d/rc.sh +++ b/ansible/roles/hosts/files/etc/profile.d/rc.sh @@ -1,11 +1,11 @@ # shellcheck shell=sh -## rc.sh calls user defined functions -# author: Yann "aya" Autissier -# license: MIT -# updated: 2021/03/04 +# file rc.sh: Call user defined functions +## author: Yann "aya" Autissier +## license: MIT +## version: 20210620 case $- in - # if we are in an interactive shell + # if this is an interactive shell *i*) # load user stuff from ~/.rc.d/* files for file in "${HOME}"/.rc.d/*; do @@ -41,3 +41,5 @@ case $- in unset IFS ;; esac + +# vim:ts=2:sw=2:sts=2:et diff --git a/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh b/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh index 8ab8678..915e853 100644 --- a/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh +++ b/ansible/roles/hosts/files/etc/profile.d/rc_functions.sh @@ -1,10 +1,10 @@ # shellcheck shell=sh -## rc_function.sh defines customs shell functions -# author: Yann "aya" Autissier -# license: MIT -# updated: 2021/03/04 +# file rc_functions.sh: Define shell functions +## author: Yann "aya" Autissier +## license: MIT +## version: 20210620 -## force() runs a command sine die +# function force: Run a command sine die force() { if [ $# -gt 0 ]; then while true; do @@ -14,7 +14,7 @@ force() { fi } -## force8() runs a command sine die if not already running +# function force8: Run a command sine die if not already running force8() { if [ $# -gt 0 ]; then while true; do @@ -38,13 +38,13 @@ force8() { fi } -## load_average() prints the current load average +# function load_average; Print the current load average load_average() { awk '{printf "%.1f\n" $1}' /proc/loadavg 2>/dev/null\ || uptime 2>/dev/null |awk '{printf "%.1f\n", $(NF-2)}' } -## process_count() prints number of "processes"/"running processes"/"D-state" +# function process_count: Print number of "processes"/"running processes"/"D-state" process_count() { ps ax -o stat 2>/dev/null |awk ' $1 ~ /R/ {process_running++}; @@ -52,7 +52,7 @@ process_count() { END { print NR-1"/"process_running+0"/"process_dstate+0; }' } -## prompt_set() exports custom PROMPT_COMMAND +# function prompt_set: Export custom PROMPT_COMMAND prompt_set() { case "${TERM}" in screen*) @@ -81,7 +81,7 @@ prompt_set() { unset ESCAPE_CODE_DCS ESCAPE_CODE_ST } -## ps1_set() exports custom PS1 +# function ps1_set: Export custom PS1 ps1_set() { case "$0" in *sh) @@ -164,7 +164,7 @@ ps1_set() { PS1_USER PS1_USER_COLOR PS1_STATUS PS1_WORKDIR } -## screen_attach() attaches existing screen session or creates a new one +# function screen_attach: Attach existing screen session or Create a new one screen_attach() { command -v screen >/dev/null 2>&1 || return SCREEN_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" @@ -182,12 +182,12 @@ screen_attach() { unset SCREEN_SESSION } -## screen_detach() detaches current screen session +# function screen_detach: Detach current screen session screen_detach() { screen -d } -## ssh_add() loads all private keys in ~/.ssh/ to ssh agent +# function ssh_add: Load all private keys in ~/.ssh/ to ssh agent ssh_add() { command -v ssh-agent >/dev/null 2>&1 && command -v ssh-add >/dev/null 2>&1 || return SSH_AGENT_DIR="/tmp/ssh-$(id -u)" @@ -224,7 +224,7 @@ ssh_add() { unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_AGENT_DIR SSH_AGENT_SOCK SSH_PRIVATE_KEYS } -## ssh_del() removes all private keys in ~/.ssh/ from ssh agent +# function ssh_del: removes all private keys in ~/.ssh/ from ssh agent ssh_del() { command -v ssh-add >/dev/null 2>&1 || return # attach to agent @@ -250,7 +250,7 @@ ssh_del() { unset GREP_RECURSIVE_CHAR GREP_RECURSIVE_FLAG SSH_PRIVATE_KEYS } -## tmux_attach() attaches existing tmux session or creates a new one +# function tmux_attach: Attach existing tmux session or Create a new one tmux_attach() { command -v tmux >/dev/null 2>&1 || return TMUX_SESSION="$(id -nu)@$(hostname |sed 's/\..*//')" @@ -263,12 +263,12 @@ tmux_attach() { unset TMUX_SESSION } -## tmux_detach() detaches current tmux session +# function tmux_detach: Detach current tmux session tmux_detach() { tmux detach } -## user_count() prints number of "users sessions"/"users"/"logged users" +# function user_count: Print number of "users sessions"/"users"/"logged users" user_count() { ps ax -o user,tty 2>/dev/null |awk ' $2 ~ /^(pts|tty)/ { users_session++; logged[$1]++; }; diff --git a/ansible/roles/hosts/files/etc/ssh/github.com.pub b/ansible/roles/hosts/files/etc/ssh/github.com.pub deleted file mode 100644 index 1bae52b..0000000 --- a/ansible/roles/hosts/files/etc/ssh/github.com.pub +++ /dev/null @@ -1 +0,0 @@ -github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== diff --git a/ansible/roles/hosts/tasks/cloudinit.yml b/ansible/roles/hosts/tasks/cloudinit.yml index 92d3fcb..8b44066 100644 --- a/ansible/roles/hosts/tasks/cloudinit.yml +++ b/ansible/roles/hosts/tasks/cloudinit.yml @@ -7,10 +7,9 @@ when: hosts_enable_cloudinit|default(false) and ansible_os_family|lower != "alpine" - name: cloudinit - install cloud-init packages - apk: name="{{item.name}}" state="{{item.state}}" apk: - name: cloud-init - state: present + name: "{{ item.name}}" + state: "{{ item.state }}" repository: - http://dl-cdn.alpinelinux.org/alpine/edge/main - http://dl-cdn.alpinelinux.org/alpine/edge/testing @@ -25,7 +24,7 @@ - name: cloudinit - update /etc/cloud/cloud.cfg template: - src: etc/cloud/cloud.cfg.j2 + src: cloud.cfg.j2 dest: /etc/cloud/cloud.cfg force: yes when: hosts_enable_cloudinit|default(false) diff --git a/ansible/roles/hosts/tasks/dir.yml b/ansible/roles/hosts/tasks/dir.yml new file mode 100644 index 0000000..7569e4d --- /dev/null +++ b/ansible/roles/hosts/tasks/dir.yml @@ -0,0 +1,15 @@ +--- +# file: tasks/dir.yml + +- name: dir - create directories + with_items: + - ~/.config + - ~/.config/git + - ~/.ssh + - ~/.ssh/myos + - ~/.tmux + - ~/.tmux/myos + file: + path: "{{item}}" + state: directory + diff --git a/ansible/roles/hosts/tasks/git.yml b/ansible/roles/hosts/tasks/git.yml index b5df36c..f06fc3c 100644 --- a/ansible/roles/hosts/tasks/git.yml +++ b/ansible/roles/hosts/tasks/git.yml @@ -8,3 +8,4 @@ dest: "{{ item.dest|default('/src') }}" key_file: "{{ item.key_file|default('~/.ssh/id_rsa') }}" version: "{{ item.version|default('HEAD') }}" + become: yes diff --git a/ansible/roles/hosts/tasks/main.yml b/ansible/roles/hosts/tasks/main.yml index c0f5f53..729ddde 100644 --- a/ansible/roles/hosts/tasks/main.yml +++ b/ansible/roles/hosts/tasks/main.yml @@ -10,21 +10,24 @@ - import_tasks: cloudinit.yml tags: - cloudinit -- import_tasks: packages.yml +- import_tasks: dir.yml tags: - - packages -- import_tasks: ssh.yml - tags: - - ssh + - dir - import_tasks: files.yml tags: - files - import_tasks: git.yml tags: - git +- import_tasks: packages.yml + tags: + - packages - import_tasks: service.yml tags: - service +- import_tasks: ssh.yml + tags: + - ssh - import_tasks: user.yml tags: - user diff --git a/ansible/roles/hosts/tasks/ssh.yml b/ansible/roles/hosts/tasks/ssh.yml index 3847434..20d26c8 100644 --- a/ansible/roles/hosts/tasks/ssh.yml +++ b/ansible/roles/hosts/tasks/ssh.yml @@ -1,24 +1,30 @@ --- # file: tasks/ssh.yml -- name: ssh - add keys to file ~/.ssh/authorized_keys - authorized_key: user="root" key=https://github.com/{{item}}.keys - with_items: "{{hosts_ssh_users|default([])}}" - become: yes +- name: ssh - add ssh_authorized_keys to file ~/.ssh/authorized_keys + authorized_key: user="{{ ansible_user|default('root') }}" key="{{ item }}" + with_items: "{{ hosts_ssh_authorized_keys|default([]) }}" + ignore_errors: true -- name: ssh - copy ssh private keys - with_items: "{{hosts_ssh_private_keys|default([])}}" - copy: src={{item}} dest=~/.ssh/ mode=0400 - become: yes - -- name: ssh - add public hosts keys to known_hosts - with_items: "{{hosts_ssh_public_hosts_keys|default([])}}" +- name: ssh - add ssh_public_hosts keys to known_hosts + with_items: "{{ hosts_ssh_public_hosts|default([]) }}" known_hosts: - name: "{{item.name}}" - key: "{{ lookup('file', '{{item.key}}') }}" - become: yes + name: "{{ item }}" + key: "{{ lookup('pipe', 'ssh-keyscan -t rsa -H ' + item) }}" + ignore_errors: true -- name: ssh - define configuration +- name: ssh - copy ssh_private_keys to ~/.ssh/ + with_items: "{{ hosts_ssh_private_keys|default([]) }}" + copy: src="{{ item }}" dest=~/.ssh/ mode=0400 + ignore_errors: true + +- name: ssh - update ~/.ssh/myos/config + template: + src: ssh_config.j2 + dest: ~/.ssh/myos/config + mode: 0400 + +- name: ssh - define sshd configuration set_fact: sshd_config: - dest: /etc/conf.d/dropbear @@ -27,22 +33,25 @@ - dest: /etc/ssh/sshd_config line: Banner /etc/issue.net regex: ^#?Banner + - dest: /etc/ssh/sshd_config + line: PermitRootLogin prohibit-password + regex: ^#?PermitRootLogin -- name: ssh - stat configuration file +- name: ssh - stat sshd configuration file changed_when: false register: sshd_config_stat stat: - path: '{{item.dest}}' - with_items: '{{sshd_config|default([])}}' + path: "{{ item.dest }}" + with_items: "{{ sshd_config|default([]) }}" - name: ssh - configure sshd become: yes lineinfile: backrefs: true - dest: '{{item.0.dest}}' - line: '{{item.0.line}}' - regex: '{{item.0.regex}}' + dest: "{{ item.0.dest }}" + line: "{{ item.0.line }}" + regex: "{{ item.0.regex }}" with_together: - - '{{sshd_config|default([])}}' - - '{{sshd_config_stat.results}}' + - "{{ sshd_config|default([]) }}" + - "{{ sshd_config_stat.results }}" when: item.1.stat.exists diff --git a/ansible/roles/hosts/tasks/user.yml b/ansible/roles/hosts/tasks/user.yml index 1c09345..c801461 100644 --- a/ansible/roles/hosts/tasks/user.yml +++ b/ansible/roles/hosts/tasks/user.yml @@ -3,7 +3,7 @@ - name: user - create ~/.env template: - src: .env.j2 + src: env.j2 dest: ~/.env force: no mode: 0400 @@ -11,40 +11,49 @@ - name: user - create ~/.rc.d file: path=~/.rc.d/ state={{hosts_enable_rc|default(false)|ternary('directory', 'absent')}} mode="0700" -- name: user - activate rc functions +- name: user - source /etc/profile.d/rc_functions.sh + with_items: + - /etc/profile.d/rc_functions.sh + lineinfile: dest=~/.rc.d/00_source create=yes line="{{ item }}" mode="0600" + when: hosts_enable_rc|default(false) + +- name: user - enable rc functions with_items: "{{hosts_rc_functions|default([])}}" file: path="~/.rc.d/{{item}}" state="touch" mode="0600" when: hosts_enable_rc|default(false) - name: user - disable rc functions with_items: "{{hosts_rc_cleanup|default([])}}" - file: path="~/.rc.d/{{item}}" state="absent" mode="0600" + file: path="~/.rc.d/{{item}}" state="absent" when: hosts_enable_rc|default(false) -- name: user - create directories +- name: user - copy ~/.*aliases ~/.*functions ~/.*profile ~/.shrc with_items: - - ~/.config - - ~/.config/git - file: - path: "{{item}}" - state: directory + - ".dircolors_aliases" + - ".docker_aliases" + - ".resty_aliases" + - ".vim_aliases" + - ".sh_functions" + - ".bash_profile" + - ".profile" + - ".shrc" + copy: src=../files/{{item}} dest=~/{{item}} mode="0640" backup="yes" + +- name: user - update ~/.bashrc + with_items: + - PS1="╭∩╮$PS1" + - unset user_count process_count + lineinfile: dest=~/.bashrc create=yes line='{{item}}' - name: user - update ~/.config/git/ignore with_items: - '.nfs*' - '*~' - '*.log' + - '*.pyc' - '*.swp' lineinfile: dest=~/.config/git/ignore create=yes line='{{item}}' -- name: user - update ~/.profile - with_items: - - alias ctop='docker run --rm -ti --volume /var/run/docker.sock:/var/run/docker.sock:ro quay.io/vektorlab/ctop:latest' - - alias vi='vim' - - export EDITOR='vim' - - export PAGER='less' - lineinfile: dest=~/.profile create=yes line='{{item}}' - - name: user - update ~/.screenrc with_items: - defscrollback 1024 @@ -52,9 +61,31 @@ - shell -$SHELL lineinfile: dest=~/.screenrc create=yes line='{{item}}' +- name: user - update ~/.ssh/config + with_items: + - include myos/config + lineinfile: dest=~/.ssh/config create=yes line='{{item}}' + +- name: user - update ~/.tmux.conf + with_items: + - source-file ~/.tmux/myos/config + lineinfile: dest=~/.tmux.conf create=yes line='{{item}}' + +- name: user - update ~/.tmux/myos/config + template: + src: tmux_config.j2 + dest: ~/.tmux/myos/config + mode: 0400 + +- name: user - update ~/.tmux/myos/session + template: + src: tmux_session.j2 + dest: ~/.tmux/myos/session + mode: 0400 + - name: user - update ~/.vimrc with_items: - - :set et ai bg=dark sw=4 ts=4 encoding=utf-8 mouse="" + - :set et ai bg=dark sw=2 ts=2 encoding=utf-8 mouse="" - :syn on - :filetype plugin indent on lineinfile: dest=~/.vimrc create=yes line='{{item}}' diff --git a/ansible/roles/hosts/templates/etc/cloud/cloud.cfg.j2 b/ansible/roles/hosts/templates/cloud.cfg.j2 similarity index 100% rename from ansible/roles/hosts/templates/etc/cloud/cloud.cfg.j2 rename to ansible/roles/hosts/templates/cloud.cfg.j2 diff --git a/ansible/roles/hosts/templates/.env.j2 b/ansible/roles/hosts/templates/env.j2 similarity index 53% rename from ansible/roles/hosts/templates/.env.j2 rename to ansible/roles/hosts/templates/env.j2 index 6ea892e..f24b6e0 100644 --- a/ansible/roles/hosts/templates/.env.j2 +++ b/ansible/roles/hosts/templates/env.j2 @@ -1,5 +1,4 @@ {% for var in hosts_user_env|default([]) %} {{var}}={{ lookup('env',var) }} {% endfor %} -ENV_USER_VARS={{hosts_user_env |default([]) |join(' ')}} DOCKER -DOCKER=false +ENV_USER_VARS={{hosts_user_env |default([]) |join(' ')}} diff --git a/ansible/roles/hosts/templates/ssh_config.j2 b/ansible/roles/hosts/templates/ssh_config.j2 new file mode 100644 index 0000000..c34012f --- /dev/null +++ b/ansible/roles/hosts/templates/ssh_config.j2 @@ -0,0 +1,22 @@ +Host * + LogLevel quiet + Compression yes +{% if hosts_ssh_bastion_hostname|length %} +Host myos-bastion + HostName {{ hosts_ssh_bastion_hostname }} +{% if hosts_ssh_bastion_username|length %} + User {{ hosts_ssh_bastion_username }} +{% endif %} +{% endif %} +{% if hosts_ssh_private_ip_range|length %} +Host {{ hosts_ssh_private_ip_range }} +{% if hosts_ssh_bastion_hostname|length %} + ProxyCommand ssh -q myos-bastion nc -q0 %h 22 +{% endif %} + HostName %h +{% if hosts_ssh_username|length %} + User {{ hosts_ssh_username }} +{% endif %} + StrictHostKeyChecking no + UserKnownHostsFile /dev/null +{% endif %} diff --git a/ansible/roles/hosts/templates/tmux_config.j2 b/ansible/roles/hosts/templates/tmux_config.j2 new file mode 100644 index 0000000..3577eb5 --- /dev/null +++ b/ansible/roles/hosts/templates/tmux_config.j2 @@ -0,0 +1,31 @@ +# this is myos's tmux configuration ! +bind r source-file ~/.tmux.conf\; display-message "tmux.conf reloaded!" +bind R move-window -r +bind C-s source ~/.tmux/myos/session +bind C-n switch-client -n\; refresh-client -S +bind C-p switch-client -p\; refresh-client -S +bind Escape copy-mode +bind Y run "tmux save-buffer - |xsel -i" +bind P run "xsel -o |tmux load-buffer -; tmux paste-buffer" +bind C-c run "tmux save-buffer - | xclip -i -sel clipboard >/dev/null" +bind C-v run "tmux set-buffer \"$(xclip -o -sel clipboard)\"; tmux paste-buffer" +bind -n S-down new-window +bind -n S-left prev +bind -n S-right next +bind -n C-left swap-window -t -1 +bind -n C-right swap-window -t +1 +set -g aggressive-resize on +set -g status-keys vi +set -g mode-keys vi +set -g monitor-activity on +set -g visual-activity on +set -g allow-rename on +set -g default-terminal "screen" +set -g history-limit 4242 +set -g status-bg black +set -g status-fg white +set -g status-interval 60 +set -g status-left-length 30 +set -g status-left '<#[fg=green]#S#[default]> ' +set -g status-right '#[fg=yellow] %d/%m %H:%M#[default]' +set -g update-environment "SSH_ASKPASS SSH_AUTH_SOCK SSH_CONNECTION" diff --git a/ansible/roles/hosts/templates/tmux_session.j2 b/ansible/roles/hosts/templates/tmux_session.j2 new file mode 100644 index 0000000..e4a2da9 --- /dev/null +++ b/ansible/roles/hosts/templates/tmux_session.j2 @@ -0,0 +1 @@ +run-shell "aws --region eu-west-1 ec2 describe-instances --no-paginate --query 'Reservations[*].Instances[*].[Tags[?Key==\`Name\`].Value,PrivateIpAddress]' --output text |sed '$!N;s/\\n/ /;/^None/d;' |awk '$2 ~ /^myos\.[0-9a-z]+\.\${USER}$/' |while read ip host; do env_user=\${host%myos.}; env=\$(host#.*); num=\$(echo \${ip} |tr . '\n' |awk '{n = n*256 + \$1} END {print n}'); tmux neww -t myos-\${env}:\${num} -n \${ip} '/bin/bash -cli \"force ssh \${ip}\"'; done" diff --git a/ansible/roles/hosts/tests/goss/bash.yml b/ansible/roles/hosts/tests/goss/bash.yml deleted file mode 100644 index 9837a4a..0000000 --- a/ansible/roles/hosts/tests/goss/bash.yml +++ /dev/null @@ -1,10 +0,0 @@ -file: - /etc/bashrc: - exists: true - mode: "0644" - owner: root - group: root - filetype: file - contains: - - "source /etc/profile.d/bashrc.sh" - diff --git a/ansible/roles/hosts/tests/goss/bash_common.yml b/ansible/roles/hosts/tests/goss/bash_common.yml deleted file mode 100644 index a571261..0000000 --- a/ansible/roles/hosts/tests/goss/bash_common.yml +++ /dev/null @@ -1,12 +0,0 @@ -file: - /etc/profile.d/bashrc.sh: - exists: true - mode: "0644" - owner: root - group: root - filetype: file - contains: - - /^function git_branch/ - - /^function process_count/ - - /^function load_average/ - diff --git a/ansible/roles/hosts/tests/goss/bash_debian.yml b/ansible/roles/hosts/tests/goss/bash_debian.yml deleted file mode 100644 index d30317b..0000000 --- a/ansible/roles/hosts/tests/goss/bash_debian.yml +++ /dev/null @@ -1,13 +0,0 @@ -file: - /etc/bash.bashrc: - exists: true - mode: "0644" - owner: root - group: root - filetype: file - contains: - - "source /etc/profile.d/bashrc.sh" - -gossfile: - bash_common.yml: {} - diff --git a/ansible/roles/hosts/tests/goss/file.yml b/ansible/roles/hosts/tests/goss/file.yml new file mode 100644 index 0000000..cfdc7a1 --- /dev/null +++ b/ansible/roles/hosts/tests/goss/file.yml @@ -0,0 +1,14 @@ +file: + /etc/profile.d/rc.sh: + exists: true + mode: "0644" + owner: root + group: root + filetype: file + /etc/profile.d/rc_functions.sh: + exists: true + mode: "0644" + owner: root + group: root + filetype: file + diff --git a/ansible/roles/hosts/tests/goss/main.yml b/ansible/roles/hosts/tests/goss/main.yml index 3b84e79..a4bb014 100644 --- a/ansible/roles/hosts/tests/goss/main.yml +++ b/ansible/roles/hosts/tests/goss/main.yml @@ -1,5 +1,4 @@ gossfile: + file.yml: {} package.yml: {} - bash.yml: {} - root.yml: {} - # ssh.yml: {} + ssh.yml: {} diff --git a/ansible/roles/hosts/tests/goss/main_alpine.yml b/ansible/roles/hosts/tests/goss/main_alpine.yml new file mode 100644 index 0000000..70c2b40 --- /dev/null +++ b/ansible/roles/hosts/tests/goss/main_alpine.yml @@ -0,0 +1,4 @@ +gossfile: + file.yml: {} + package_alpine.yml: {} + ssh.yml: {} diff --git a/ansible/roles/hosts/tests/goss/main_debian.yml b/ansible/roles/hosts/tests/goss/main_debian.yml index 2081ec2..058d193 100644 --- a/ansible/roles/hosts/tests/goss/main_debian.yml +++ b/ansible/roles/hosts/tests/goss/main_debian.yml @@ -1,5 +1,4 @@ gossfile: + file.yml: {} package_debian.yml: {} - bash_debian.yml: {} - root.yml: {} - # ssh.yml: {} + ssh.yml: {} diff --git a/ansible/roles/hosts/tests/goss/main_redhat.yml b/ansible/roles/hosts/tests/goss/main_redhat.yml index f23fcda..e472ddb 100644 --- a/ansible/roles/hosts/tests/goss/main_redhat.yml +++ b/ansible/roles/hosts/tests/goss/main_redhat.yml @@ -1,5 +1,4 @@ gossfile: + file.yml: {} package_redhat.yml: {} - bash.yml: {} - root.yml: {} - # ssh.yml: {} + ssh.yml: {} diff --git a/ansible/roles/hosts/tests/goss/package.yml b/ansible/roles/hosts/tests/goss/package.yml index 952530d..d430ee7 100644 --- a/ansible/roles/hosts/tests/goss/package.yml +++ b/ansible/roles/hosts/tests/goss/package.yml @@ -1,5 +1,25 @@ -gossfile: - package_common.yml: {} package: - vim: + bash: + installed: true + ca-certificates: + installed: true + curl: + installed: true + git: + installed: true + htop: + installed: true + less: + installed: true + lsof: + installed: true + make: + installed: true + rsync: + installed: true + screen: + installed: true + tmux: + installed: true + tzdata: installed: true diff --git a/ansible/roles/hosts/tests/goss/package_common.yml b/ansible/roles/hosts/tests/goss/package_alpine.yml similarity index 55% rename from ansible/roles/hosts/tests/goss/package_common.yml rename to ansible/roles/hosts/tests/goss/package_alpine.yml index add35fb..99c781b 100644 --- a/ansible/roles/hosts/tests/goss/package_common.yml +++ b/ansible/roles/hosts/tests/goss/package_alpine.yml @@ -1,11 +1,13 @@ +gossfile: + package.yml: {} package: - bash: + coreutils: installed: true - ca-certificates: + groff: installed: true - screen: + openssh-client: installed: true - rsync: + util-linux: installed: true - tzdata: + vim: installed: true diff --git a/ansible/roles/hosts/tests/goss/package_debian.yml b/ansible/roles/hosts/tests/goss/package_debian.yml index 29f9168..d9b16f1 100644 --- a/ansible/roles/hosts/tests/goss/package_debian.yml +++ b/ansible/roles/hosts/tests/goss/package_debian.yml @@ -1,5 +1,13 @@ gossfile: - package_common.yml: {} + package.yml: {} package: + coreutils: + installed: true + groff: + installed: true + openssh-client: + installed: true + util-linux: + installed: true vim-nox: installed: true diff --git a/ansible/roles/hosts/tests/goss/package_redhat.yml b/ansible/roles/hosts/tests/goss/package_redhat.yml index dbc647c..576da5d 100644 --- a/ansible/roles/hosts/tests/goss/package_redhat.yml +++ b/ansible/roles/hosts/tests/goss/package_redhat.yml @@ -1,5 +1,11 @@ gossfile: - package_common.yml: {} + package.yml: {} package: + groff-base: + installed: true + libselinux-python: + installed: true + openssh-clients: + installed: true vim-minimal: installed: true diff --git a/ansible/roles/hosts/tests/goss/root.yml b/ansible/roles/hosts/tests/goss/root.yml deleted file mode 100644 index 76ff600..0000000 --- a/ansible/roles/hosts/tests/goss/root.yml +++ /dev/null @@ -1,9 +0,0 @@ -file: - /root/.screenrc: - exists: true - mode: "0644" - owner: root - group: root - filetype: file - contains: - - /^hardstatus alwayslastline/ diff --git a/ansible/roles/hosts/tests/goss/ssh.yml b/ansible/roles/hosts/tests/goss/ssh.yml index c9a0600..2a3574a 100644 --- a/ansible/roles/hosts/tests/goss/ssh.yml +++ b/ansible/roles/hosts/tests/goss/ssh.yml @@ -7,11 +7,3 @@ file: filetype: file contains: - /^PermitRootLogin prohibit-password/ - /root/.ssh/authorized_keys: - exists: true - mode: "0600" - owner: root - group: root - filetype: file - contains: - - "Jpb0EeFEebgvi7Kpp6gpIXKFEeuuE" diff --git a/ansible/roles/hosts/vars/alpine.yml b/ansible/roles/hosts/vars/alpine.yml new file mode 100644 index 0000000..39db34b --- /dev/null +++ b/ansible/roles/hosts/vars/alpine.yml @@ -0,0 +1,10 @@ +--- +# file vars/alpine.yml + +hosts_packages_distro: + - { "name": "coreutils", "state": "present" } + - { "name": "groff", "state": "present" } + - { "name": "openssh-client", "state": "present" } + - { "name": "util-linux", "state": "present" } + - { "name": "vim", "state": "present" } + diff --git a/ansible/roles/hosts/vars/debian.yml b/ansible/roles/hosts/vars/debian.yml index e1dcd19..36b8e78 100644 --- a/ansible/roles/hosts/vars/debian.yml +++ b/ansible/roles/hosts/vars/debian.yml @@ -2,5 +2,9 @@ # file vars/debian.yml hosts_packages_distro: + - { "name": "coreutils", "state": "present" } + - { "name": "groff", "state": "present" } + - { "name": "openssh-client", "state": "present" } + - { "name": "util-linux", "state": "present" } - { "name": "vim-nox", "state": "present" } diff --git a/ansible/roles/hosts/vars/fedora-5.yml b/ansible/roles/hosts/vars/fedora-5.yml index 3a6de01..67f9931 100644 --- a/ansible/roles/hosts/vars/fedora-5.yml +++ b/ansible/roles/hosts/vars/fedora-5.yml @@ -1,4 +1,4 @@ hosts_packages_distro: [] hosts_packages_common: [] -hosts_ssh_users: [] +hosts_ssh_authorized_keys: [] diff --git a/ansible/roles/hosts/vars/redhat.yml b/ansible/roles/hosts/vars/redhat.yml index 904498d..9c015a8 100644 --- a/ansible/roles/hosts/vars/redhat.yml +++ b/ansible/roles/hosts/vars/redhat.yml @@ -2,6 +2,8 @@ # file vars/redhat.yml hosts_packages_distro: - - { "name": "vim", "state": "present" } + - { "name": "groff-base", "state": "present" } - { "name": "libselinux-python", "state": "present" } + - { "name": "openssh-clients", "state": "present" } + - { "name": "vim-minimal", "state": "present" } diff --git a/aws/aws.mk b/aws/aws.mk index 29fdf64..024e2f6 100644 --- a/aws/aws.mk +++ b/aws/aws.mk @@ -17,7 +17,7 @@ aws-deploy: .PHONY: aws-docker-login aws-docker-login: aws-ecr-get-login -# target aws-ecr-get-login: Exec 'Call aws ecr get-login' +# target aws-ecr-get-login: Call aws ecr get-login .PHONY: aws-ecr-get-login aws-ecr-get-login: $(eval IGNORE_DRYRUN := true) diff --git a/docker/cli/Dockerfile b/docker/cli/Dockerfile index 887ea11..5b365f8 100644 --- a/docker/cli/Dockerfile +++ b/docker/cli/Dockerfile @@ -43,7 +43,7 @@ ARG SHELL=/bin/bash ARG SSH_BASTION_HOSTNAME ARG SSH_BASTION_USERNAME ARG SSH_PRIVATE_IP_RANGE -ARG SSH_PUBLIC_HOST_KEYS +ARG SSH_PUBLIC_HOSTS ARG UID ARG USER ENV UID=${UID} @@ -110,7 +110,7 @@ WORKDIR /home/$USER # git config RUN mkdir -p ~/.ssh ~/.config/git \ - && ssh-keyscan -t rsa -H ${SSH_PUBLIC_HOST_KEYS} >> ~/.ssh/known_hosts \ + && ssh-keyscan -t rsa -H ${SSH_PUBLIC_HOSTS} >> ~/.ssh/known_hosts \ && echo -e "\ .DS_Store\n\ .idea/\n\ diff --git a/docker/registrator/Dockerfile b/docker/registrator/Dockerfile index 3b5d6ed..1eb44e0 100644 --- a/docker/registrator/Dockerfile +++ b/docker/registrator/Dockerfile @@ -14,16 +14,17 @@ WORKDIR /go/src/github.com/gliderlabs/registrator/ RUN \ apk add --no-cache curl git \ && git clone https://github.com/gliderlabs/registrator/ . \ - && git reset --hard da90d170da9dd7e1a8d9a13429d44686dc3d118f \ - # -useIpFromNetwork command line option \ - && git fetch origin pull/596/head \ - && git merge --no-edit 8d904c60949e310893a25c8af3636b0151334dd4 \ - # convert check.Script to check.Args \ - && git fetch origin pull/627/head \ - && git merge --no-edit ed053c364e3ba941aeca9ab0d8791b051ff4dede \ - # skip tls verification \ - && git fetch origin pull/661/head \ - && git merge --no-edit 38fc83ac07b4a070be71079cb810429d94a60205 \ +# TEMPORARY DISABLED - TODO: check with new upstream master +# && git reset --hard da90d170da9dd7e1a8d9a13429d44686dc3d118f \ +# # -useIpFromNetwork command line option \ +# && git fetch origin pull/596/head \ +# && git merge --no-edit 8d904c60949e310893a25c8af3636b0151334dd4 \ +# # convert check.Script to check.Args \ +# && git fetch origin pull/627/head \ +# && git merge --no-edit ed053c364e3ba941aeca9ab0d8791b051ff4dede \ +# # skip tls verification \ +# && git fetch origin pull/661/head \ +# && git merge --no-edit 38fc83ac07b4a070be71079cb810429d94a60205 \ && curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh \ && dep ensure -vendor-only \ && CGO_ENABLED=0 GOOS=linux go build \ diff --git a/docker/theia/Dockerfile b/docker/theia/Dockerfile index 91be08a..10bd762 100644 --- a/docker/theia/Dockerfile +++ b/docker/theia/Dockerfile @@ -166,7 +166,7 @@ ARG UID ARG GID ARG SSH_BASTION_HOSTNAME ARG SSH_BASTION_USERNAME -ARG SSH_PUBLIC_HOST_KEYS +ARG SSH_PUBLIC_HOSTS ARG SSH_PRIVATE_IP_RANGE ARG USER ENV UID=${UID} @@ -237,7 +237,7 @@ RUN yarn --cache-folder ./ycache && rm -rf ./ycache \ # git config RUN mkdir -p ~/.ssh ~/.config/git \ - && ssh-keyscan -t rsa -H ${SSH_PUBLIC_HOST_KEYS} >> ~/.ssh/known_hosts \ + && ssh-keyscan -t rsa -H ${SSH_PUBLIC_HOSTS} >> ~/.ssh/known_hosts \ && echo -e "\ .DS_Store\n\ .idea/\n\ diff --git a/make/apps/common.mk b/make/apps/common.mk index 9656a55..1399f53 100644 --- a/make/apps/common.mk +++ b/make/apps/common.mk @@ -9,20 +9,11 @@ bootstrap: bootstrap-git bootstrap-docker app-bootstrap ## Update application fi # target bootstrap-docker: Build and start application dockers # on local host .PHONY: boostrap-docker -bootstrap-docker: docker-network-create - $(call make,docker-compose-up) +bootstrap-docker: install-bin-docker docker-network-create # target bootstrap-git: Fire update-app .PHONY: bootstrap-git -bootstrap-git: update-app - -# target bootstrap-git-%: Clone GIT_REPOSITORY in folder % -# on local host -.PHONY: bootstrap-git-% -bootstrap-git-%: - if ! git config remote.origin.url > /dev/null ; \ - then git clone $(QUIET) $(GIT_REPOSITORY) $*; \ - fi +bootstrap-git: install-bin-git # target build: Build application docker images to run # on local host @@ -45,7 +36,7 @@ build@%: myos-base # target clean: Clean application and docker images # on local host .PHONY: clean app-clean -clean: app-clean docker-rm docker-image-rm docker-volume-rm .env-clean ## Clean application and docker stuffs +clean: app-clean docker-rm docker-images-rm docker-volume-rm .env-clean ## Clean application and docker stuffs # target clean@%: Clean deployed application and docker images of % ENV # on local host @@ -98,7 +89,7 @@ exec@%: # target install app-install: Install application # on local host .PHONY: install app-install -install: update-app app-install ## Install application +install: bootstrap app-install ## Install application # target logs: Display application dockers logs # on local host diff --git a/make/apps/def.build.mk b/make/apps/def.build.mk index 47b2b50..e801f83 100644 --- a/make/apps/def.build.mk +++ b/make/apps/def.build.mk @@ -1,8 +1,8 @@ BUILD_AUTHOR ?= $(DOCKER_AUTHOR) BUILD_DATE ?= $(shell TZ=UTC date "+%Y%m%dT%H%M%SZ" 2>/dev/null) BUILD_DESCRIPTION ?= Lot of Love -BUILD_DOCUMENTATION ?= $(if $(wildcard README.md),$(APP_REPOSITORY)/blob/$(COMMIT)/README.md) -BUILD_ENV_VARS ?= APP BRANCH BUILD_DATE BUILD_STATUS COMMIT DEPLOY_HOOK_URL ENV VERSION +BUILD_DOCUMENTATION ?= $(if $(wildcard README.md),$(APP_REPOSITORY_URL)/blob/$(COMMIT)/README.md) +BUILD_ENV_VARS ?= APP BRANCH BUILD_DATE BUILD_STATUS COMMIT DEPLOY_HOOK_URL ENV UID USER VERSION BUILD_LABEL_VARS ?= org.label-schema.% org.opencontainers.% os.my.% BUILD_LABEL_ARGS ?= $(foreach var,$(filter $(BUILD_LABEL_VARS),$(MAKE_FILE_VARS)),$(if $($(var)),$(var)='$($(var))')) BUILD_LICENSE ?= GPL-3.0 @@ -19,17 +19,17 @@ org.label-schema.schema-version ?= 1.0 org.label-schema.url ?= $(APP_URL) org.label-schema.usage ?= $(BUILD_DOCUMENTATION) org.label-schema.vcs-ref ?= $(COMMIT) -org.label-schema.vcs-url ?= $(APP_REPOSITORY) +org.label-schema.vcs-url ?= $(APP_REPOSITORY_URL) org.label-schema.vendor ?= $(BUILD_AUTHOR) org.label-schema.version ?= $(VERSION) org.opencontainers.image.created ?= $(BUILD_DATE) org.opencontainers.image.revision ?= $(COMMIT) -org.opencontainers.image.source ?= $(APP_REPOSITORY) +org.opencontainers.image.source ?= $(APP_REPOSITORY_URL) org.opencontainers.image.url ?= $(APP_URL) org.opencontainers.image.vendor ?= $(BUILD_AUTHOR) org.opencontainers.image.version ?= $(VERSION) org.opencontainers.image.url ?= $(APP_URL) -org.opencontainers.image.source ?= $(APP_REPOSITORY) +org.opencontainers.image.source ?= $(APP_REPOSITORY_URL) org.opencontainers.image.version ?= $(VERSION) org.opencontainers.image.revision ?= $(COMMIT) org.opencontainers.image.vendor ?= $(BUILD_AUTHOR) diff --git a/make/apps/def.docker.mk b/make/apps/def.docker.mk index be9cdbc..2c746d2 100644 --- a/make/apps/def.docker.mk +++ b/make/apps/def.docker.mk @@ -47,7 +47,7 @@ DOCKER_SERVICES ?= $(eval IGNORE_DRYRUN := true)$(shell $(call d DOCKER_SHELL ?= $(SHELL) ENV_VARS += COMPOSE_PROJECT_NAME COMPOSE_SERVICE_NAME DOCKER_BUILD_TARGET DOCKER_GID DOCKER_IMAGE_TAG DOCKER_REGISTRY DOCKER_REPOSITORY DOCKER_SHELL -ifeq ($(DOCKER), true) +ifneq ($(DOCKER_RUN),) DOCKER_COMPOSE ?= docker/compose:$(COMPOSE_VERSION) else DOCKER_COMPOSE ?= $(or $(shell docker compose >/dev/null 2>&1 && printf 'docker compose\n'),docker-compose) diff --git a/make/apps/def.mk b/make/apps/def.mk index 0912f9b..cf0b6fd 100644 --- a/make/apps/def.mk +++ b/make/apps/def.mk @@ -5,13 +5,19 @@ APP_HOST ?= $(APP)$(addprefix .,$(APP_DOMAIN)) APP_INSTALLED ?= $(APPS) APP_PARENT ?= $(MONOREPO) APP_PARENT_DIR ?= $(MONOREPO_DIR) -APP_PATH ?= /$(APP_PATH_PREFIX) -APP_REPOSITORY ?= $(GIT_REPOSITORY) +APP_PATH += $(APP_PATH_PREFIX) +APP_REPOSITORY ?= $(APP_REPOSITORY_URL) +APP_REPOSITORY_HOST ?= $(shell printf '$(APP_REPOSITORY_URI)\n' |sed 's|/.*||;s|.*@||') +APP_REPOSITORY_PATH ?= $(shell printf '$(APP_REPOSITORY_URI)\n' |sed 's|[^/]*/||;') +APP_REPOSITORY_SCHEME ?= $(shell printf '$(APP_REPOSITORY_URL)\n' |sed 's|://.*||;') +APP_REPOSITORY_URI ?= $(shell printf '$(APP_REPOSITORY_URL)\n' |sed 's|.*://||;') +APP_REPOSITORY_URL ?= $(GIT_REPOSITORY) APP_REQUIRED ?= $(APP_REPOSITORY) APP_SCHEME ?= https APP_UPSTREAM_REPOSITORY ?= $(or $(shell git config --get remote.upstream.url 2>/dev/null),$(GIT_UPSTREAM_REPOSITORY)) -APP_URI ?= $(APP_HOST)$(APP_PATH) +APP_URI ?= $(APP_HOST)/$(APP_PATH) APP_URL ?= $(APP_SCHEME)://$(APP_URI) +CONTEXT += APP APPS BRANCH DOMAIN VERSION RELEASE CONTEXT_DEBUG += APP_DIR APP_URL APP_REPOSITORY APP_UPSTREAM_REPOSITORY ENV_DEPLOY ENV_DEPLOY ?= $(shell ls .git/refs/remotes/origin/ 2>/dev/null) ENV_VARS += APP_DIR APP_DOMAIN APP_HOST APP_PATH APP_URL CONSUL_HTTP_TOKEN $(if $(filter true,$(MOUNT_NFS)),NFS_CONFIG) diff --git a/make/apps/docker.mk b/make/apps/docker.mk index bb60545..52c2692 100644 --- a/make/apps/docker.mk +++ b/make/apps/docker.mk @@ -122,7 +122,7 @@ docker-images-rm: docker-images-rm-%: docker images |awk '$$1 ~ /^$(subst /,\/,$*)/ {print $$3}' |sort -u |while read image; do $(RUN) docker rmi -f $$image; done -# target docker-login: Exec 'docker login' +# target docker-login: Run 'docker login' .PHONY: docker-login docker-login: myos-base $(RUN) docker login @@ -131,7 +131,7 @@ docker-login: myos-base .PHONY: docker-network-create docker-network-create: docker-network-create-$(DOCKER_NETWORK) -# target docker-network-create-%: Exec 'docker network create %' +# target docker-network-create-%: Run 'docker network create %' .PHONY: docker-network-create-% docker-network-create-%: if [ -z "$(shell docker network ls -q --filter name='^$*$$' 2>/dev/null)" ]; then \ @@ -142,18 +142,25 @@ docker-network-create-%: .PHONY: docker-network-rm docker-network-rm: docker-network-rm-$(DOCKER_NETWORK) -# target docker-network-rm-%: Remove docker network % +# target docker-network-rm-%: Run 'docker network rm %' .PHONY: docker-network-rm-% docker-network-rm-%: if [ -n "$(shell docker network ls -q --filter name='^$*$$' 2>/dev/null)" ]; then \ $(RUN) docker network rm $* >/dev/null \ && $(or $(call INFO,docker network $* removed), true); fi \ -# target docker-plugin-install: Exec 'docker plugin install DOCKER_PLUGIN_OPTIONS DOCKER_PLUGIN' +# target docker-plugin-install: Run 'docker plugin install DOCKER_PLUGIN_OPTIONS DOCKER_PLUGIN' .PHONY: docker-plugin-install docker-plugin-install: $(eval docker_plugin_state := $(shell docker plugin ls | awk '$$2 == "$(DOCKER_PLUGIN)" {print $$NF}') ) - $(if $(docker_plugin_state),$(if $(filter $(docker_plugin_state),false),printf "Enabling docker plugin $(DOCKER_PLUGIN) ... " && $(RUN) docker plugin enable $(DOCKER_PLUGIN) >/dev/null 2>&1 && printf "done\n" || printf "ERROR\n"),printf "Installing docker plugin $(DOCKER_PLUGIN) ... " && $(RUN) docker plugin install $(DOCKER_PLUGIN_OPTIONS) $(DOCKER_PLUGIN) $(DOCKER_PLUGIN_ARGS) >/dev/null 2>&1 && printf "done\n" || printf "ERROR\n") + $(if $(docker_plugin_state),\ + $(if $(filter $(docker_plugin_state),false),\ + $(RUN) docker plugin enable $(DOCKER_PLUGIN) >/dev/null 2>&1 \ + && $(or $(call INFO,docker plugin $(DOCKER_PLUGIN) enabled), true) \ + ), \ + $(RUN) docker plugin install $(DOCKER_PLUGIN_OPTIONS) $(DOCKER_PLUGIN) $(DOCKER_PLUGIN_ARGS) >/dev/null 2>&1 \ + && $(or $(call INFO,docker plugin $(DOCKER_PLUGIN) installed), true) \ + ) # target docker-push: Call docker-push for each SERVICES .PHONY: docker-push @@ -205,7 +212,7 @@ docker-run-%: docker-build-% $(eval path := $(patsubst %/,%,$*)) $(eval image := $(DOCKER_REPOSITORY)/$(lastword $(subst /, ,$(path)))$(if $(findstring :,$*),,:$(DOCKER_IMAGE_TAG))) $(eval image_id := $(shell docker images -q $(image) 2>/dev/null)) - $(call docker-run,$(command),$(if $(image_id),$(image),$(path))) + $(call docker-run,$(RUN) $(command),$(if $(image_id),$(image),$(path))) # target docker-tag: Call docker-tag for each SERVICES .PHONY: docker-tag diff --git a/make/apps/install.mk b/make/apps/install.mk index 89d41de..5cae306 100644 --- a/make/apps/install.mk +++ b/make/apps/install.mk @@ -9,11 +9,11 @@ .PHONY: install-mysql-database-% install-mysql-database-%: $(if $(DOCKER_RUN),myos-base) $(call exec,mysql -h mysql -u root -proot $* -e "use $*" >/dev/null 2>&1) \ - || $(RUN) $(call exec,mysql -h mysql -u root -proot mysql -e "create database $* character set utf8 collate utf8_unicode_ci;") + || $(call exec,$(RUN) mysql -h mysql -u root -proot mysql -e "create database $* character set utf8 collate utf8_unicode_ci;") $(call exec,mysql -h mysql -u $* -p$* $* -e "use $*" >/dev/null 2>&1) \ - || $(RUN) $(call exec,mysql -h mysql -u root -proot mysql -e "grant all privileges on $*.* to '$*'@'%' identified by '$*'; flush privileges;") + || $(call exec,$(RUN) mysql -h mysql -u root -proot mysql -e "grant all privileges on $*.* to '$*'@'%' identified by '$*'; flush privileges;") $(call exec,sh -c '[ $$(mysql -h mysql -u $* -p$* $* -e "show tables" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.mysql.gz" ]') \ - && $(RUN) $(call exec,sh -c 'gzip -cd "${APP_DIR}/$*.mysql.gz" |mysql -h mysql -u root -proot $*') \ + && $(call exec,$(RUN) sh -c 'gzip -cd "${APP_DIR}/$*.mysql.gz" |mysql -h mysql -u root -proot $*') \ ||: # target install-pgsql-database-%: Import %.pgsql.gz to database % @@ -24,14 +24,14 @@ install-mysql-database-%: $(if $(DOCKER_RUN),myos-base) .PHONY: install-pgsql-database-% install-pgsql-database-%: myos-base $(call exec,PGPASSWORD=$* psql -h postgres -U $* template1 -c "\q" >/dev/null 2>&1) \ - || $(RUN) $(call exec,PGPASSWORD=postgres psql -h postgres -U postgres -c "create user $* with createdb password '$*';") + || $(call exec,$(RUN) PGPASSWORD=postgres psql -h postgres -U postgres -c "create user $* with createdb password '$*';") $(call exec,PGPASSWORD=$* psql -h postgres -U $* -d $* -c "" >/dev/null 2>&1) \ - || $(RUN) $(call exec,PGPASSWORD=postgres psql -h postgres -U postgres -c "create database $* owner $* ;") + || $(call exec,$(RUN) PGPASSWORD=postgres psql -h postgres -U postgres -c "create database $* owner $* ;") $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql.gz" ]) \ - && $(RUN) $(call exec,sh -c 'gzip -cd "${APP_DIR}/$*.pgsql.gz" |PGPASSWORD="postgres" psql -h postgres -U postgres -d $*') \ + && $(call exec,$(RUN) sh -c 'gzip -cd "${APP_DIR}/$*.pgsql.gz" |PGPASSWORD="postgres" psql -h postgres -U postgres -d $*') \ ||: $(call exec,[ $$(PGPASSWORD=$* psql -h postgres -U $* -d $* -c "\d" 2>/dev/null |wc -l) -eq 0 ] && [ -f "${APP_DIR}/$*.pgsql" ]) \ - && $(RUN) $(call exec,sh -c 'PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH SUPERUSER" && PGPASSWORD="postgres" pg_restore -h postgres --no-owner --role=$* -U postgres -d $* ${APP_DIR}/$*.pgsql && PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH NOSUPERUSER"') \ + && $(call exec,$(RUN) sh -c 'PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH SUPERUSER" && PGPASSWORD="postgres" pg_restore -h postgres --no-owner --role=$* -U postgres -d $* ${APP_DIR}/$*.pgsql && PGPASSWORD="postgres" psql -h postgres -U postgres -c "ALTER ROLE $* WITH NOSUPERUSER"') \ ||: # target install-build-config: Call install-config with file * and dest build diff --git a/make/apps/myos/def.mk b/make/apps/myos/def.mk index b165737..30f315b 100644 --- a/make/apps/myos/def.mk +++ b/make/apps/myos/def.mk @@ -4,6 +4,6 @@ ENV_VARS += COMPOSE_IGNORE_ORPHANS DOCKER_IMAGE_CLI DOCKE HOME ?= /home/$(USER) NFS_DISK ?= $(NFS_HOST):/$(notdir $(SHARED)) NFS_OPTIONS ?= rw,rsize=8192,wsize=8192,bg,hard,intr,nfsvers=3,noatime,nodiratime,actimeo=3 -NFS_PATH ?= /srv/$(subst :,,$(NFS_DISK)) +NFS_PATH ?= /srv/$(subst $(space),/,$(strip $(call reverse,$(subst ., ,$(NFS_HOST)))))$(subst ..,,$(SHARED)) SHELL ?= /bin/sh STACK ?= base diff --git a/make/apps/myos/def.ssh.mk b/make/apps/myos/def.ssh.mk index d5a8a8d..cb71469 100644 --- a/make/apps/myos/def.ssh.mk +++ b/make/apps/myos/def.ssh.mk @@ -1,11 +1,15 @@ DOCKER_BUILD_VARS += $(SSH_ENV_VARS) ENV_VARS += $(SSH_ENV_VARS) +SSH_AUTHORIZED_KEYS ?= $(SSH_GITHUB_AUTHORIZED_KEYS) SSH_BASTION_HOSTNAME ?= -SSH_BASTION_USERNAME ?= -SSH_ENV_VARS ?= SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PUBLIC_HOST_KEYS SSH_PRIVATE_IP_RANGE -SSH_PUBLIC_HOST_KEYS ?= $(SSH_REMOTE_HOSTS) $(SSH_BASTION_HOSTNAME) +SSH_BASTION_USERNAME ?= $(SSH_USER) +SSH_ENV_VARS ?= SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PUBLIC_HOSTS SSH_PRIVATE_IP_RANGE SSH_USER +SSH_GITHUB_AUTHORIZED_KEYS ?= $(patsubst %,https://github.com/%,$(patsubst %,%.keys,$(SSH_USER))) +SSH_PUBLIC_HOSTS ?= $(CONFIG_REPOSITORY_HOST) $(SSH_BASTION_HOSTNAME) $(SSH_REMOTE_HOSTS) SSH_PRIVATE_IP_RANGE ?= +SSH_PRIVATE_KEYS ?= $(wildcard $(SSH_DIR)/id_rsa) SSH_REMOTE_HOSTS ?= github.com gitlab.com +SSH_USER ?= $(call slugify,$(GIT_USER)) # function ssh-connect: Exec command 2 on remote hosts 1 with tty define ssh-connect diff --git a/make/common.mk b/make/common.mk index c92baaf..1f20d58 100644 --- a/make/common.mk +++ b/make/common.mk @@ -17,6 +17,11 @@ install-app install-apps: myos-base install-app-required install-app-required: myos-base $(foreach url,$(APP_REQUIRED),$(call install-app,$(url))) +# target install-bin-%; Call ansible-run-localhost when bin % is not available +.PHONY: install-bin-% +install-bin-%:; + $(if $(shell type $* 2>/dev/null),,$(call make,ansible-run-localhost)) + # target $(SHARED): Create SHARED folder $(SHARED): $(RUN) mkdir -p $(SHARED) @@ -36,8 +41,6 @@ update-app-%: % ; # target update-config: Update config files .PHONY: update-config -update-config: SSH_PUBLIC_HOST_KEYS := $(CONFIG_REMOTE_HOST) $(SSH_BASTION_HOSTNAME) $(SSH_REMOTE_HOSTS) -update-config: MAKE_VARS += SSH_BASTION_HOSTNAME SSH_BASTION_USERNAME SSH_PRIVATE_IP_RANGE SSH_PUBLIC_HOST_KEYS update-config: myos-base $(call update-app,$(CONFIG_REPOSITORY),$(CONFIG)) @@ -47,27 +50,32 @@ update-config: myos-base .PHONY: update-hosts update-hosts: ifneq (,$(filter $(ENV),local)) - cat .env */.env 2>/dev/null |grep -Eo 'urlprefix-[^/]+' |sed 's/urlprefix-//' |while read host; do grep $$host /etc/hosts >/dev/null 2>&1 || { printf "Adding $$host to /etc/hosts\n"; printf "127.0.0.1 $$host\n" |$(RUN) sudo tee -a /etc/hosts >/dev/null; }; done + cat .env */.env 2>/dev/null |grep -Eo 'urlprefix-[^/]+' |sed 's/urlprefix-//' |while read host; do \ + grep $$host /etc/hosts >/dev/null 2>&1 || { \ + printf "Adding $$host to /etc/hosts\n"; \ + printf "127.0.0.1 $$host\n" |$(RUN) sudo tee -a /etc/hosts >/dev/null; \ + }; \ + done endif # target update-remote-%: fetch git remote % .PHONY: update-remote-% update-remote-%: myos-base - $(RUN) $(call exec,git fetch --prune --tags $*) + $(RUN) git fetch --prune --tags $* # target update-remotes: fetch all git remotes .PHONY: update-remotes update-remotes: myos-base - $(RUN) $(call exec,git fetch --all --prune --tags) + $(RUN) git fetch --all --prune --tags # target update-upstream: fetch git remote upstream .PHONY: update-upstream update-upstream: myos-base .git/refs/remotes/upstream/master - $(RUN) $(call exec,git fetch --prune --tags upstream) + $(RUN) git fetch --prune --tags upstream -# target .git/refs/remotes/upstream/master: git add upstream APP_UPSTREAM_REPOSITORY +# target .git/refs/remotes/upstream/master: add git upstream APP_UPSTREAM_REPOSITORY .git/refs/remotes/upstream/master: - $(RUN) git remote add upstream $(APP_UPSTREAM_REPOSITORY) 2>/dev/null ||: + $(RUN) git remote add upstream $(APP_UPSTREAM_REPOSITORY) # target shared: Fire SHARED .PHONY: update-shared diff --git a/make/def.docker.mk b/make/def.docker.mk index f29e9a2..323f4f8 100644 --- a/make/def.docker.mk +++ b/make/def.docker.mk @@ -14,7 +14,6 @@ DOCKER_NETWORK_PRIVATE ?= $(USER_ENV) DOCKER_NETWORK_PUBLIC ?= node DOCKER_REPOSITORY_MYOS ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_MYOS)) DOCKER_REPOSITORY_NODE ?= $(subst _,/,$(COMPOSE_PROJECT_NAME_NODE)) -DOCKER_RUN ?= $(filter true,$(DOCKER)) # DOCKER_RUN_OPTIONS: default options of `docker run` command DOCKER_RUN_OPTIONS += --rm -it # DOCKER_RUN_VOLUME: options -v of `docker run` command to mount additionnal volumes @@ -33,7 +32,7 @@ else DOCKER_RUN_VOLUME += -v $(or $(APP_PARENT_DIR),$(APP_DIR),$(PWD)):$(or $(WORKSPACE_DIR),$(APP_PARENT_DIR),$(APP_DIR),$(PWD)) endif -ifeq ($(DOCKER), true) +ifneq ($(DOCKER_RUN),) DOCKER_SSH_AUTH := -e SSH_AUTH_SOCK=/tmp/ssh-agent/socket -v $(DOCKER_VOLUME_SSH):/tmp/ssh-agent diff --git a/make/def.mk b/make/def.mk index 90aec90..1da235d 100644 --- a/make/def.mk +++ b/make/def.mk @@ -12,6 +12,10 @@ APP_TYPE ?= $(if $(SUBREPO),subrepo) $(if $(filter .,$(MY APPS ?= $(if $(MONOREPO),$(sort $(patsubst $(MONOREPO_DIR)/%/.git,%,$(wildcard $(MONOREPO_DIR)/*/.git)))) APPS_NAME ?= $(foreach app,$(APPS),$(or $(shell awk -F '=' '$$1 == "APP" {print $$2}' $(or $(wildcard $(MONOREPO_DIR)/$(app)/.env),$(wildcard $(MONOREPO_DIR)/$(app)/.env.$(ENV)),$(MONOREPO_DIR)/$(app)/.env.dist) 2>/dev/null),$(app))) BRANCH ?= $(GIT_BRANCH) +CMD_APK_INSTALL ?= $(if $(shell type -p apk),apk --no-cache --update add) +CMD_APK_REMOVE ?= $(if $(shell type -p apk),apk --no-cache del) +CMD_APT_INSTALL ?= $(if $(shell type -p apt-get),apt-get update && apt-get -fy install) +CMD_APT_REMOVE ?= $(if $(shell type -p apt-get),apt-get -fy remove) CMDS ?= exec exec:% exec@% install-app install-apps run run:% run@% COLOR_INFO ?= $(COLOR_BROWN) COLOR_HIGHLIGHT ?= $(COLOR_GREEN) @@ -28,17 +32,24 @@ COLOR_CYAN ?= \033[36m COLOR_GRAY ?= \033[37m COMMIT ?= $(or $(SUBREPO_COMMIT),$(GIT_COMMIT)) CONFIG ?= $(RELATIVE)config -CONFIG_REPOSITORY ?= $(call pop,$(or $(APP_UPSTREAM_REPOSITORY),$(GIT_UPSTREAM_REPOSITORY)))/$(notdir $(CONFIG)) -CONTEXT ?= $(if $(APP),APP BRANCH DOMAIN VERSION) $(shell awk 'BEGIN {FS="="}; $$1 !~ /^(\#|$$)/ {print $$1}' .env.dist 2>/dev/null) +CONFIG_REPOSITORY ?= $(CONFIG_REPOSITORY_URL) +CONFIG_REPOSITORY_HOST ?= $(shell printf '$(CONFIG_REPOSITORY_URI)\n' |sed 's|/.*||;s|.*@||') +CONFIG_REPOSITORY_PATH ?= $(shell printf '$(CONFIG_REPOSITORY_URI)\n' |sed 's|[^/]*/||;') +CONFIG_REPOSITORY_SCHEME ?= $(shell printf '$(CONFIG_REPOSITORY_URL)\n' |sed 's|://.*||;') +CONFIG_REPOSITORY_URI ?= $(shell printf '$(CONFIG_REPOSITORY_URL)\n' |sed 's|.*://||;') +CONFIG_REPOSITORY_URL ?= $(call pop,$(APP_UPSTREAM_REPOSITORY))/$(notdir $(CONFIG)) +CONTEXT ?= ENV $(shell awk 'BEGIN {FS="="}; $$1 !~ /^(\#|$$)/ {print $$1}' .env.dist 2>/dev/null) CONTEXT_DEBUG ?= MAKEFILE_LIST env env.docker APPS GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME MAKE_DIR MAKE_SUBDIRS MAKE_CMD_ARGS MAKE_ENV_ARGS UID USER DEBUG ?= -DOCKER ?= $(if $(BUILD),false,true) +DOCKER ?= $(shell type -p docker) +DOCKER_RUN ?= $(if $(filter-out false False FALSE,$(DOCKER)),$(DOCKER)) DOMAIN ?= localhost DRONE ?= false DRYRUN ?= false DRYRUN_RECURSIVE ?= false ELAPSED_TIME = $(shell $(call TIME)) ENV ?= local +ENV_ARGS = $(if $(DOCKER_RUN),$(env.docker.args) $(env.docker.dist),$(env.args) $(env.dist)) ENV_FILE ?= $(wildcard $(CONFIG)/$(ENV)/$(APP)/.env .env) ENV_LIST ?= $(shell ls .git/refs/heads/ 2>/dev/null) ENV_RESET ?= false @@ -53,10 +64,14 @@ GIT_STATUS ?= $(shell git status -uno --porcelain 2>/dev/nu GIT_TAG ?= $(shell git tag -l --points-at $(BRANCH) 2>/dev/null) GIT_UPSTREAM_REPOSITORY ?= $(if $(findstring ://,$(GIT_REPOSITORY)),$(call pop,$(call pop,$(GIT_REPOSITORY)))/,$(call pop,$(GIT_REPOSITORY),:):)$(GIT_UPSTREAM_USER)/$(lastword $(subst /, ,$(GIT_REPOSITORY))) GIT_UPSTREAM_USER ?= $(lastword $(subst /, ,$(call pop,$(MYOS_REPOSITORY)))) +GIT_USER ?= $(GIT_AUTHOR_NAME) GIT_VERSION ?= $(shell git describe --tags $(BRANCH) 2>/dev/null || git rev-parse $(BRANCH) 2>/dev/null) HOSTNAME ?= $(shell hostname 2>/dev/null |sed 's/\..*//') IGNORE_DRYRUN ?= false IGNORE_VERBOSE ?= false +INSTALL ?= $(SUDO) $(subst &&,&& $(SUDO),$(INSTALL_CMD)) +INSTALL_CMDS ?= APK_INSTALL APT_INSTALL +$(foreach cmd,$(INSTALL_CMDS),$(if $(CMD_$(cmd)),$(eval INSTALL_CMD ?= $(CMD_$(cmd))))) LOG_LEVEL ?= $(if $(DEBUG),debug,$(if $(VERBOSE),info,error)) MAKE_ARGS ?= $(foreach var,$(MAKE_VARS),$(if $($(var)),$(var)='$($(var))')) MAKE_SUBDIRS ?= $(if $(filter myos,$(MYOS)),monorepo,$(if $(APP),apps $(foreach type,$(APP_TYPE),$(if $(wildcard $(MAKE_DIR)/apps/$(type)),apps/$(type))))) @@ -83,6 +98,7 @@ SHARED ?= $(RELATIVE)shared SSH_DIR ?= ${HOME}/.ssh STATUS ?= $(GIT_STATUS) SUBREPO ?= $(if $(wildcard .gitrepo),$(notdir $(CURDIR))) +SUDO ?= $(if $(filter-out 0,$(UID)),$(shell type -p sudo)) TAG ?= $(GIT_TAG) UID ?= $(shell id -u 2>/dev/null) USER ?= $(shell id -nu 2>/dev/null) @@ -90,18 +106,13 @@ USER_ENV ?= $(USER)_$(ENV) VERBOSE ?= $(if $(DEBUG),true) VERSION ?= $(GIT_VERSION) -ifeq ($(DOCKER), true) -ENV_ARGS = $(env.docker.args) $(env.docker.dist) -else -ENV_ARGS = $(env.args) $(env.dist) -endif - ifneq ($(DEBUG),) CONTEXT += $(CONTEXT_DEBUG) else .SILENT: endif +# Guess RUN ifeq ($(DRYRUN),true) RUN = $(if $(filter-out true,$(IGNORE_DRYRUN)),echo) ifeq ($(RECURSIVE), true) @@ -124,14 +135,12 @@ HOST_SYSTEM := DARWIN endif endif -# include .env files -include $(wildcard $(ENV_FILE)) - ifeq ($(HOST_SYSTEM),DARWIN) -ifneq ($(DOCKER),true) SED_SUFFIX := '' endif -endif + +# include .env files +include $(wildcard $(ENV_FILE)) # function conf: Extract variable=value line from configuration files ## it prints the line with variable 3 definition from block 2 in file 1 @@ -165,7 +174,20 @@ endef # macro force: Run command 1 sine die ## it starts command 1 if it is not already running ## it returns never -force = $$(while true; do [ $$(ps x |awk 'BEGIN {nargs=split("'"$$*"'",args)} $$field == args[1] { matched=1; for (i=1;i<=NF-field;i++) { if ($$(i+field) == args[i+1]) {matched++} } if (matched == nargs) {found++} } END {print found+0}' field=4) -eq 0 ] && $(RUN) $(1) || sleep 1; done) +force = $$(while true; do \ + [ $$(ps x |awk '\ + BEGIN {nargs=split("'"$$*"'",args)} \ + $$field == args[1] { \ + matched=1; \ + for (i=1;i<=NF-field;i++) { \ + if ($$(i+field) == args[i+1]) {matched++} \ + } \ + if (matched == nargs) {found++} \ + } \ + END {print found+0}' field=4) -eq 0 \ + ] \ + && $(RUN) $(1) || sleep 1; done \ +) # macro gid: Return GID of group 1 gid = $(shell grep '^$(1):' /etc/group 2>/dev/null |awk -F: '{print $$3}') @@ -188,14 +210,14 @@ ${COLOR_HIGHLIGHT}$@${COLOR_RESET}:${COLOR_RESET} ' >&$(INFO_FD) \ && printf '\n' >&$(INFO_FD) \ )) -# function install-app: Exec 'git clone url 1 dir 2' or Call update-app with url 1 dir 2 +# function install-app: Run 'git clone url 1 dir 2' or Call update-app with url 1 dir 2 define install-app $(call INFO,install-app,$(1)$(comma) $(2)) - $(eval url := $(or $(1), $(APP_REPOSITORY))) + $(eval url := $(or $(1), $(APP_REPOSITORY_URL))) $(eval dir := $(or $(2), $(RELATIVE)$(lastword $(subst /, ,$(url))))) $(if $(wildcard $(dir)/.git), \ - $(call update-app,$(url),$(dir)), \ - $(RUN) $(call exec,git clone $(QUIET) $(url) $(dir)) \ + $(call update-app,$(url),$(dir)), \ + $(RUN) git clone $(QUIET) $(url) $(dir) \ ) endef @@ -228,21 +250,21 @@ endef # macro pop: Return last word of string 1 according to separator 2 pop = $(patsubst %$(or $(2),/)$(lastword $(subst $(or $(2),/), ,$(1))),%,$(1)) -# macro sed: Exec sed script 1 on file 2 -sed = $(call env-exec,$(RUN) sed -i $(SED_SUFFIX) '$(1)' $(2)) +# macro sed: Run sed script 1 on file 2 +sed = $(RUN) sed -i $(SED_SUFFIX) '$(1)' $(2) # macro TIME: Print time elapsed since unixtime 1 TIME = awk '{printf "%02d:%02d:%02d\n",int($$1/3600),int(($$1%3600)/60),int($$1%60)}' \ - <<< $(shell bc <<< "$(or $(2),$(MAKE_UNIXTIME_CURRENT))-$(or $(1),$(MAKE_UNIXTIME_START))" 2>/dev/null) + <<< $(shell awk 'BEGIN {current=$(or $(2),$(MAKE_UNIXTIME_CURRENT)); start=$(or $(1),$(MAKE_UNIXTIME_START)); print (current - start)}' 2>/dev/null) -# function update-app: Exec 'cd dir 1 && git pull' or Call install-app +# function update-app: Run 'cd dir 1 && git pull' or Call install-app define update-app $(call INFO,update-app,$(1)$(comma) $(2)) - $(eval url := $(or $(1), $(APP_REPOSITORY))) + $(eval url := $(or $(1), $(APP_REPOSITORY_URL))) $(eval dir := $(or $(2), $(APP_DIR))) $(if $(wildcard $(dir)/.git), \ - $(RUN) $(call exec,sh -c 'cd $(dir) && git pull $(QUIET)'), \ - $(call install-app,$(url),$(dir)) \ + $(RUN) sh -c 'cd $(dir) && git pull $(QUIET)', \ + $(call install-app,$(url),$(dir)) \ ) endef diff --git a/make/env.mk b/make/env.mk index 3a846ea..558730b 100644 --- a/make/env.mk +++ b/make/env.mk @@ -90,5 +90,5 @@ define .env_update awk '{while(match($$0,"[$$]{[^}]*}")) {var=substr($$0,RSTART+2,RLENGTH-3);gsub("[$$]{"var"}",ENVIRON[var])} print}') \ |sed -e /^$$/d -e /^#/d \ |sort \ - >> $(env_file) + >> $(env_file); endef diff --git a/make/monorepo/def.mk b/make/monorepo/def.mk index 69959de..36635de 100644 --- a/make/monorepo/def.mk +++ b/make/monorepo/def.mk @@ -1,5 +1,5 @@ CMDS += copy master-tag release release-check release-create release-finish subrepo-push subrepo-update -CONTEXT += APPS ENV RELEASE +CONTEXT += APPS DOMAIN RELEASE DIRS ?= $(CONFIG) $(MAKE_DIR) $(SHARED) RELEASE_UPGRADE ?= $(filter v%, $(shell git tag -l 2>/dev/null |sort -V |awk '/$(RELEASE)/,0')) RELEASE_VERSION ?= $(firstword $(subst -, ,$(VERSION))) diff --git a/make/monorepo/subrepo.mk b/make/monorepo/subrepo.mk index d829f03..fa9e037 100644 --- a/make/monorepo/subrepo.mk +++ b/make/monorepo/subrepo.mk @@ -3,10 +3,10 @@ # target subrepo-branch-delete: Delete branch $(BRANCH) on remote $(SUBREPO) .PHONY: subrepo-branch-delete -subrepo-branch-delete: $(if $(DOCKER_RUN),myos-base) subrepo-check +subrepo-branch-delete: myos-base subrepo-check ifneq ($(words $(BRANCH)),0) - $(call exec,[ $$(git ls-remote --heads $(REMOTE) $(BRANCH) 2>/dev/null |wc -l) -eq 1 ]) \ - && $(call exec,$(RUN) git push $(REMOTE) :$(BRANCH)) + [ $$(git ls-remote --heads $(REMOTE) $(BRANCH) 2>/dev/null |wc -l) -eq 1 ] \ + && $(RUN) git push $(REMOTE) :$(BRANCH) endif # target subrepo-check: Define SUBREPO and REMOTE @@ -28,21 +28,21 @@ endif .PHONY: subrepo-git-diff subrepo-git-diff: myos-base subrepo-check $(eval IGNORE_DRYRUN := true) - $(eval DIFF = $(shell $(call exec,git diff --quiet $(shell $(call exec,git rev-list --ancestry-path $(shell awk '$$1 == "parent" {print $$3}' $(SUBREPO)/.gitrepo)..HEAD |tail -n 1)) -- $(SUBREPO); printf '$$?\n')) ) + $(eval DIFF = $(shell git diff --quiet $(shell git rev-list --ancestry-path $(shell awk '$$1 == "parent" {print $$3}' $(SUBREPO)/.gitrepo)..HEAD |tail -n 1) -- $(SUBREPO); printf '$$?\n') ) $(eval IGNORE_DRYRUN := false) # target subrepo-git-fetch: Fetch git remote .PHONY: subrepo-git-fetch subrepo-git-fetch: myos-base subrepo-check - $(call exec,$(RUN) git fetch --prune $(REMOTE)) + $(RUN) git fetch --prune $(REMOTE) # target subrepo-tag-create-%: Create tag TAG to reference branch REMOTE/% .PHONY: subrepo-tag-create-% subrepo-tag-create-%: myos-base subrepo-check subrepo-git-fetch ifneq ($(words $(TAG)),0) - $(call exec,[ $$(git ls-remote --tags $(REMOTE) $(TAG) |wc -l) -eq 0 ]) \ - || $(call exec,$(RUN) git push $(REMOTE) :refs/tags/$(TAG)) - $(call exec,$(RUN) git push $(REMOTE) refs/remotes/subrepo/$(SUBREPO)/$*:refs/tags/$(TAG)) + [ $$(git ls-remote --tags $(REMOTE) $(TAG) |wc -l) -eq 0 ] \ + || $(call exec,$(RUN) git push $(REMOTE) :refs/tags/$(TAG)) + $(RUN) git push $(REMOTE) refs/remotes/subrepo/$(SUBREPO)/$*:refs/tags/$(TAG) endif # target subrepo-push: Push to subrepo @@ -55,7 +55,7 @@ endif # if specific branch name, delete remote branch before push and recreate it from master ifneq ($(findstring $(firstword $(subst /, ,$(BRANCH))),feature hotfix release story),) $(eval IGNORE_DRYRUN := true) - $(eval DELETE = $(shell $(call exec,sh -c 'git ls-remote --heads $(REMOTE) $(BRANCH) |wc -l')) ) + $(eval DELETE = $(shell sh -c 'git ls-remote --heads $(REMOTE) $(BRANCH) |wc -l') ) $(eval IGNORE_DRYRUN := false) else $(eval DELETE = 0) @@ -64,12 +64,12 @@ endif $(call INFO,subrepo $(SUBREPO) already up to date); \ else \ if [ $(DELETE) -eq 1 ]; then \ - $(call exec,$(RUN) git push $(REMOTE) :$(BRANCH)); \ - $(call exec,$(RUN) git push $(REMOTE) refs/remotes/$(REMOTE)/master:refs/heads/$(BRANCH)); \ + $(RUN) git push $(REMOTE) :$(BRANCH); \ + $(RUN) git push $(REMOTE) refs/remotes/$(REMOTE)/master:refs/heads/$(BRANCH); \ fi; \ - $(call exec,$(RUN) git subrepo fetch $(SUBREPO) -b $(BRANCH)); \ - $(call exec,$(RUN) git subrepo push $(SUBREPO) -b $(BRANCH) $(UPDATE_SUBREPO_OPTIONS)); \ - $(call exec,$(RUN) git subrepo clean $(SUBREPO)); \ + $(RUN) git subrepo fetch $(SUBREPO) -b $(BRANCH); \ + $(RUN) git subrepo push $(SUBREPO) -b $(BRANCH) $(UPDATE_SUBREPO_OPTIONS); \ + $(RUN) git subrepo clean $(SUBREPO); \ fi # target subrepos-branch-delete: Fire APPS target @@ -83,7 +83,7 @@ subrepos-tag-create-%: $(APPS) ; # target subrepos-update: Fire APPS target and push updates to upstream .PHONY: subrepos-update subrepos-update: myos-base git-stash $(APPS) git-unstash ## Update subrepos - $(call exec,$(RUN) git push upstream $(BRANCH)) + $(RUN) git push upstream $(BRANCH) # target subrepo-update-%: Call subrepo-update target in folder % .PHONY: subrepo-update-% diff --git a/make/utils.mk b/make/utils.mk index 774e566..0f88b9e 100644 --- a/make/utils.mk +++ b/make/utils.mk @@ -69,6 +69,7 @@ reverse = $(if $(1),$(call reverse,$(wordlist 2,$(words $(1)),$(1))) $(firstword # and in rules. Particularly useful for VCS version strings, that can contain # slashes, colons (OK in filenames but not in rules), and spaces. sanitize = $(subst $(space),_,$(subst :,_,$(subst /,_,$(strip $(1))))) +slugify = $(subst $(space),,$(subst :,,$(subst /,,$(subst .,,$(1))))) # MESSAGE Macro -- display a message in bold type MESSAGE = echo "$(TERM_BOLD)>>> $($(PKG)_NAME) $($(PKG)_VERSION) $(call qstrip,$(1))$(TERM_RESET)" diff --git a/packer/def.packer.ansible.mk b/packer/def.packer.ansible.mk deleted file mode 100644 index 6259592..0000000 --- a/packer/def.packer.ansible.mk +++ /dev/null @@ -1,4 +0,0 @@ -PACKER_BUILD_VARS += ansible_extra_vars ansible_user ansible_verbose -ansible_extra_vars ?= $(patsubst target=%,target=default,$(ANSIBLE_EXTRA_VARS)) -ansible_user ?= $(ANSIBLE_USERNAME) -ansible_verbose ?= $(ANSIBLE_VERBOSE) diff --git a/packer/def.packer.mk b/packer/def.packer.mk index c8feeb6..f5e9819 100644 --- a/packer/def.packer.mk +++ b/packer/def.packer.mk @@ -6,6 +6,7 @@ PACKER_ARCH ?= $(PACKER_ALPINE_ARCH) PACKER_BOOT_WAIT ?= 11s PACKER_BUILD_ARGS ?= -on-error=cleanup $(foreach var,$(PACKER_BUILD_VARS),$(if $($(var)),-var $(var)='$($(var))')) PACKER_BUILD_VARS += accelerator boot_wait hostname iso_name iso_size output password qemuargs ssh_wait_timeout template username +PACKER_BUILD_VARS += ansible_extra_vars ansible_user ansible_verbose PACKER_CACHE_DIR ?= build/cache PACKER_HOSTNAME ?= $(PACKER_TEMPLATE) PACKER_ISO_DATE ?= $(shell stat -c %y $(PACKER_ISO_FILE) 2>/dev/null) @@ -41,6 +42,9 @@ PACKER_BUILD_ARGS += -var ssh_port_max=$(PACKER_SSH_PORT) -var vnc endif accelerator ?= $(PACKER_QEMU_ACCELERATOR) +ansible_extra_vars ?= $(patsubst target=%,target=default,$(ANSIBLE_EXTRA_VARS)) +ansible_user ?= $(PACKER_USERNAME) +ansible_verbose ?= $(ANSIBLE_VERBOSE) boot_wait ?= $(PACKER_BOOT_WAIT) hostname ?= $(PACKER_HOSTNAME) iso_name ?= $(PACKER_ISO_NAME) @@ -73,8 +77,8 @@ endif # function packer: Call run packer with arg 1 ## it needs an empty local ssh agent (ssh-add -D) -## it needs ANSIBLE_SSH_PRIVATE_KEYS set to a key giving access to ANSIBLE_GIT_REPOSITORY without password -## it needs ANSIBLE_AWS_ACCESS_KEY_ID and ANSIBLE_AWS_SECRET_ACCESS_KEY +## it needs SSH_PRIVATE_KEYS to get access without password to GIT_REPOSITORY +## it needs AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when deploying to AWS define packer $(RUN) $(call run,packer $(1),$(DOCKER_RUN_OPTIONS_PACKER) $(DOCKER_REPOSITORY)/) endef @@ -110,4 +114,4 @@ define packer-build echo 'user: $(username)' >> $(PACKER_ISO_INFO) endef -arrays_of_dquoted_args = [ $(subst $(dquote) $(dquote),$(dquote)$(comma) $(dquote),$(subst $(dquote) $(dquote)-,$(dquote) ]$(comma) [ $(dquote)-,$(patsubst %,$(dquote)%$(dquote),$1))) ] +arrays_of_dquoted_args = $(subst $(dquote) $(dquote),$(dquote)$(comma) $(dquote),$(subst $(dquote) $(dquote)-,$(dquote) ]$(comma) [ $(dquote)-,$(patsubst %,$(dquote)%$(dquote),$1))) diff --git a/stack/base.mk b/stack/base.mk index c3f855c..22e606e 100644 --- a/stack/base.mk +++ b/stack/base.mk @@ -1,3 +1,3 @@ -# target base: Fire docker-network-create stack-base-up base-ssh-add +# target base: Fire ssh-add .PHONY: base -base: docker-network-create $(if $(DOCKER_RUN),stack-base-up) ssh-add +base: $(if $(DOCKER_RUN),install-bin-docker docker-network-create stack-base-up) ssh-add diff --git a/stack/base/base.yml b/stack/base/base.yml index 3c6925b..b728ddd 100644 --- a/stack/base/base.yml +++ b/stack/base/base.yml @@ -5,6 +5,13 @@ services: build: args: - DOCKER_BUILD_DIR=docker/cli + - GID=${GID} + - UID=${UID} + - USER=${USER} + - SSH_BASTION_HOSTNAME=${SSH_BASTION_HOSTNAME} + - SSH_BASTION_USERNAME=${SSH_BASTION_USERNAME} + - SSH_PUBLIC_HOSTS=${SSH_PUBLIC_HOSTS} + - SSH_PRIVATE_IP_RANGE=${SSH_RIVATE_IP_RANGE} context: ../.. dockerfile: docker/cli/Dockerfile target: ${DOCKER_BUILD_TARGET} @@ -26,6 +33,9 @@ services: build: args: - DOCKER_BUILD_DIR=docker/ssh + - GID=${GID} + - UID=${UID} + - USER=${USER} context: ../.. dockerfile: docker/ssh/Dockerfile target: ${DOCKER_BUILD_TARGET}