diff --git a/.env b/.env index 044749d..e970aed 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ -# A key that has access to your edx-platform Github repos and to the GCP project you want to deploy the devstack in. +# (PLEASE CHANGE) A key that has access to your edx-platform Github repos and to the GCP project you want to deploy the devstack in. SSH_KEY="$(HOME)/.ssh/id_rsa" -# DON'T CHANGE: This is the project where the your instance will be created. +# (PLEASE CHANGE) This is the project where the your instance will be created. PROJECT_ID="your-gcp-project-id" # This will be used while SSHing into your remote machine. Usually matches the one in the SSH Key. @@ -16,13 +16,13 @@ INSTANCE_NAME=$(USER)-at-$(subst .,-,$(HOST_NAME)) # The name of the image on GCP if you are going to create one. IMAGE_NAME=devstack-$(INSTANCE_NAME) -# DON'T CHANGE: A unique tag for your GCP instance. +# (DON'T CHANGE) A unique tag for your GCP instance. INSTANCE_TAG=devstack-$(INSTANCE_NAME) -# DON'T CHANGE: The name of the deny rule on GCP firewall. +# (DON'T CHANGE) The name of the deny rule on GCP firewall. DENY_FIREWALL=deny-$(INSTANCE_NAME)-webserver-access -# DON'T CHANGE: The name of the allow rule on GCP firewall. +# (DON'T CHANGE) The name of the allow rule on GCP firewall. ALLOW_FIREWALL=allow-$(INSTANCE_NAME)-webserver-access # The size of the disk your instance is going to use in GB. We don't recommend it to be less than 50. @@ -43,26 +43,40 @@ TMP_DIR=~/.devstack # The mount location. MOUNT_DIR=$(TMP_DIR)/mnt/ -# DON'T CHANGE: The hosts file location. You can change it for test purposes. +# (DON'T CHANGE) The hosts file location. You can change it for test purposes. HOSTS_FILE=/etc/hosts -# The host name you are going to use to access Tahoe (i.e. http://$(TAHOE_HOST_NAME):19000). -TAHOE_HOST_NAME=tahoe +# A comma-separated list of hostnames you'd like to use. +EDX_HOST_NAMES=edx.devstack.lms # Verbosity of GCloud commands we are running. Options [debug, info, warning, error, critical, none] VERBOSITY=critical # The location the toolkit is directing Ansible output to. To unhide it use `/dev/tty` instead. -ANSIBLE_OUTPUT=/dev/null +SHELL_OUTPUT=/dev/null + +HOME_DIR=/home/$(USER_NAME) # The location of devstack on the remote machine -DEVSTACK_WORK_DIR=~/workspace +DEVSTACK_WORK_DIR=$(HOME_DIR)/workspace # Devstack image family IMAGE_FAMILY=devstack -# This is the service account's email address that is provisioned during creation or granted to you by an admin. +# (PLEASE CHANGE) This is the service account's email address that is provisioned during creation or granted to you by an admin. SERVICE_ACCOUNT_EMAIL=my-service-account@project-id.iam.gserviceaccount.com -# The downloaded JSON file key of your GCP project. +# (PLEASE CHANGE) The downloaded JSON file key of your GCP project. SERVICE_KEY_PATH=/path/to/service/account.json + +# The devstack repository URL. Change this if you want to use another clone. +DEVSTACK_REPO_URL=https://github.com/edx/devstack.git + +# The devstack branch you want to deploy. +DEVSTACK_REPO_BRANCH=master + +# The command that runs all of your devstack services (LMS, Studio, ...) Change it if you have a custom one. +DEVSTACK_RUN_COMMAND=dev.up + +# Whether to deny all IPs (except yours) from accessing your instance or not. +RESTRICT_INSTANCE=true \ No newline at end of file diff --git a/Makefile b/Makefile index 382c4c2..1d85150 100644 --- a/Makefile +++ b/Makefile @@ -2,189 +2,100 @@ include .env* export $(shell sed 's/=.*//' .env*) SHELL := /bin/bash +VERSION = 1.0.0 .PHONY: help - -help: ## This help message - @echo -e "$$(grep -hE '^\S+:.*##' $(MAKEFILE_LIST) | \ - sed -e 's/:.*##\s*/:/' -e 's/^\(.\+\):\(.*\)/\\x1b[36m\1\\x1b[m:\2/' | column -c2 -t -s :)" - -environment.debug: ## Prints the values of the environemnt variables to be used in the make command as define in .env.* files - @echo ALLOW_FIREWALL = $(ALLOW_FIREWALL) - @echo ANSIBLE_OUTPUT = $(ANSIBLE_OUTPUT) - @echo DENY_FIREWALL = $(DENY_FIREWALL) - @echo DEVSTACK_WORK_DIR = $(DEVSTACK_WORK_DIR) - @echo DISK_SIZE = $(DISK_SIZE) - @echo HOST_NAME = $(HOST_NAME) - @echo HOSTS_FILE = $(HOSTS_FILE) - @echo IMAGE_FAMILY = $(IMAGE_FAMILY) - @echo IMAGE_NAME = $(IMAGE_NAME) - @echo INSTANCE_NAME = $(INSTANCE_NAME) - @echo INSTANCE_TAG = $(INSTANCE_TAG) - @echo INVENTORY = $(INVENTORY) - @echo MACHINE_TYPE = $(MACHINE_TYPE) - @echo MOUNT_DIR = $(MOUNT_DIR) - @echo PROJECT_ID = $(PROJECT_ID) - @echo SERVICE_ACCOUNT_EMAIL = $(SERVICE_ACCOUNT_EMAIL) - @echo SERVICE_KEY_PATH = $(SERVICE_KEY_PATH) - @echo SSH_KEY = $(SSH_KEY) - @echo TAHOE_HOST_NAME = $(TAHOE_HOST_NAME) - @echo TMP_DIR = $(TMP_DIR) - @echo USER_NAME = $(USER_NAME) - @echo VERBOSITY = $(VERBOSITY) - @echo ZONE = $(ZONE) - -environment.create: - @echo Creating \`.env.$(USER_NAME)\` file... - @[ -f .env.$(USER_NAME) ] && \ - echo ERROR: \`.env.$(USER_NAME)\` already exists! || \ - sed '/^#/! s/\(.*\)/#\1/g' <.env > .env.$(USER_NAME) +bold = \033[1m +inverted = \033[7m +underline = \033[4m +normal = \033[0m + +cyan = \033[36m +dim = \033[90m +green = \033[92m +red = \033[31m +redbold = \033[1;31m +magenta = \033[35m +yellow = \033[33m + +help: ## This help message. + @echo -e "\n\ + Sultan ${cyan}v$(VERSION)${noraml}\n\ + An Open edX Remote Devstack Toolkit by Appsembler\n\n\n\n\ + ${bold}Main Targets${normal}\n\ + =======================================================================================================\n\n\ + $$(grep -hE '^\S+:.*###' $(MAKEFILE_LIST) | sed -e 's/:.*###\s*/:/' -e 's/^\(.\+\):\(.*\)/\1:\2/' | column -c2 -t -s :)\ + \n\n\ + ${bold}All Targets${normal} \n\ + =======================================================================================================\n\n\ + $$(grep -hE '^\S+:.*##' $(MAKEFILE_LIST) | sed -e 's/:.*##\s*/:/' -e 's/^\(.\+\):\(.*\)/\1:\2/' | column -c2 -t -s :)" \ + | less ve/bin/ansible-playbook: requirements.txt @echo Installing project requirements... - @virtualenv ve - @ve/bin/pip install -r requirements.txt + @virtualenv ve &> $(SHELL_OUTPUT) + @ve/bin/pip install -r requirements.txt &> $(SHELL_OUTPUT) @make local.inventory.config -clean: ## Clean software and directory caches +include targets/*.mk + +error: + @echo '' + @echo -e "${magenta}An error happened while executing the command you just used!" + @echo -e "While this might be an issue with the tool, we would like you to do a little bit more debugging:" + @echo -e " * Run ${underline}${cyan}make environment.debug${normal}${magenta} and check if all of your environment variables hold the correct values." + @echo -e " * Toggle the verbosity settings (${bold}VERBOSITY${normal}${magenta}, and ${bold}SHELL_OUTPUT${normal}${magenta}) in your env file. Follow instructions in the comments above of them for more details." + @echo -e " * Check https://github.com/appsembler/sultan/wiki for a detailed documentation on the configuration process." + @echo -e "\nIf you couldn't identify the cause of the problem, please submit an issue on https://github.com/appsembler/sultan/issues.${normal}" + +environment.create: ### Creates a custom environment file for you where you can personalize your instance's default settings. + @echo -e "Creating your custom environment file... ${dim}(.env.$(USER_NAME))${normal}" + @[ -f .env.$(USER_NAME) ] && \ + echo -e "${yellow}The file \${bold}.env.$(USER_NAME)\${normal}${yellow} already exists! ${bold}(ABORTED)${normal}" || \ + (sed '/^#/! s/\(.*\)/#\1/g' <.env > .env.$(USER_NAME) && \ + echo -e "${green}Your env file has been successfully created.${normal}" &&\ + echo -e "Make sure to override the following variables before proceeding to the setup:" && \ + echo -e " * SSH_KEY" && \ + echo -e " * PROJECT_ID" && \ + echo -e " * SERVICE_ACCOUNT_EMAIL" && \ + echo -e " * SERVICE_KEY_PATH" \ + ) + +clean: ## Clean software and directory caches. @echo Flush pip packages... @rm -rf ve - @rm dynamic-inventory/gce.ini + @rm dynamic-inventory/gce.ini || echo '' @make ve/bin/ansible-playbook @echo Flush Ansible cache... - @. ve/bin/activate; ansible-playbook local.yml --check --flush-cache &> $(ANSIBLE_OUTPUT) - -instance.ping: ve/bin/ansible-playbook ## Performs a ping to your instance. - @. ve/bin/activate; ansible -i $(INVENTORY) $(INSTANCE_NAME) -m ping - -instance.deploy: ve/bin/ansible-playbook ## Deploys your remote instance and prepare it for devstack provisioning. - @. ve/bin/activate; ansible-playbook devstack.yml \ - -i $(INVENTORY) \ - -e "instance_name=$(INSTANCE_NAME)" - @echo Run \`make devstack.provision\` to run the devstack. - -devstack.provision: ## Provisions the devstack on your instance. - make instance.run command="cd $(DEVSTACK_WORK_DIR)/devstack/ && make dev.provision" - @echo Run \`make devstack.run\` to run the devstack. - -instance.delete: local.hosts.revert instance.firewall.deny.delete instance.firewall.allow.delete ## Deletes your instance from GCP. - @echo Removing your instance \($(INSTANCE_NAME)\) from GCP... - @gcloud compute instances delete $(INSTANCE_NAME) \ - --quiet \ - --zone=$(ZONE) \ - --verbosity $(VERBOSITY) \ - --project $(PROJECT_ID) \ - || echo 'No previous instance found' - -instance.start: ## Starts your stopped instance on GCP. - @gcloud compute instances start $(INSTANCE_NAME) \ - --zone=$(ZONE) \ - --project $(PROJECT_ID) - @make local.hosts.update - @make local.ssh.config - -instance.stop: local.hosts.revert ## Stops your instance on GCP, but doesn't delete it. - @echo Stopping your instance \($(INSTANCE_NAME)\) on GCP... - @gcloud compute instances stop $(INSTANCE_NAME) \ - --zone=$(ZONE) \ - --project $(PROJECT_ID) - -instance.create: ## Creates an empty instance for you on GCP. - @echo Creating your virtual machine on GCP... - @gcloud compute instances create $(INSTANCE_NAME) \ - --image-family=ubuntu-1804-lts \ - --image-project=gce-uefi-images \ - --boot-disk-size=$(DISK_SIZE) \ - --machine-type=$(MACHINE_TYPE) \ - --tags=devstack,http-server,$(INSTANCE_TAG) \ - --zone=$(ZONE) \ - --verbosity $(VERBOSITY) \ - --project=$(PROJECT_ID) - -instance.image.delete.command: - @echo Removing $(NAME) image from GCP... - @gcloud compute images delete $(NAME) \ - --project=$(PROJECT_ID) \ - --verbosity $(VERBOSITY) \ - --quiet \ - || echo 'No previous image found' - -instance.image.delete: ## Deletes your image from GCP. - @make NAME=$(IMAGE_NAME) instance.image.delete.command - -instance.image.master.delete: - @make NAME=$(IMAGE_FAMILY) instance.image.delete.command - -instance.image.create.command: - @gcloud beta compute images create $(NAME) \ - --source-disk=$(INSTANCE_NAME) \ - --source-disk-zone=$(ZONE) \ - --family=$(IMAGE_FAMILY) \ - --labels=user=$(INSTANCE_NAME) \ - --project=$(PROJECT_ID) - -instance.image.create: instance.image.delete instance.stop ## Creates an image from your instance on GCP. - @echo Create a new image for you on GCP... - @make NAME=$(IMAGE_NAME) instance.image.create.command - -instance.image.master.create: instance.stop instance.image.master.delete ## Creates a master image from your instance on GCP. - @echo Create a new master devstack image on GCP... - @make NAME=$(IMAGE_FAMILY) instance.image.create.command - -instance.firewall.deny.delete: ## Deletes the GCP Firewall's rule that prevents accessing your instance by all ways. - @echo Removing DENY firewall rule from gcp... - @gcloud compute firewall-rules delete $(DENY_FIREWALL) \ - --project=$(PROJECT_ID) \ - --verbosity $(VERBOSITY) \ - --quiet \ - || echo 'No previous deny firewall found' - -instance.firewall.deny.create: ## Creates a GCP Firewall's rule to prevent all kind of access to your instance. - @echo Creating DENY firewall rule in gcp... - @gcloud compute firewall-rules create $(DENY_FIREWALL) \ - --action=deny \ - --direction=ingress \ - --rules=tcp \ - --source-ranges=0.0.0.0/0 \ - --priority=1000 \ - --target-tags=$(INSTANCE_TAG) \ - --project=$(PROJECT_ID) - -instance.firewall.allow.delete: ## Deletes the GCP Firewall's rule that allows your IP to access your instance. - @echo Removing ALLOW firewall rule from gcp... - @gcloud compute firewall-rules delete $(ALLOW_FIREWALL) \ - --project=$(PROJECT_ID) \ - --verbosity $(VERBOSITY) \ - --quiet \ - || echo 'No previous allow firewall found' - -instance.firewall.allow.create: ## Creates a GCP Firewall's rule allowing your IP to access your instance. - $(eval MY_PUBLIC_IP := $(shell curl ifconfig.me)) - - @echo Creating ALLOW firewall rule in gcp... - @gcloud compute firewall-rules create $(ALLOW_FIREWALL) \ - --action allow \ - --direction ingress \ - --rules tcp \ - --source-ranges $(MY_PUBLIC_IP) \ - --priority 50 \ - --target-tags=$(INSTANCE_TAG)\ - --project=$(PROJECT_ID) - -instance.firewall.deny.refresh: instance.firewall.deny.delete instance.firewall.deny.create ## Refreshes the deny rule on GCP Firewall by deleting the old rule and creating a new one. - @echo "Deny rule has been updated on the firewall." - -instance.firewall.allow.refresh: instance.firewall.allow.delete instance.firewall.allow.create ## Refreshes the allow rule on GCP Firewall by deleting the old rule and creating a new one. - @echo "Allow rule has been updated on the firewall." - -instance.restrict: instance.firewall.deny.refresh instance.firewall.allow.refresh ## Restricts the access to your instance to you only by creating the necessary rules. - @echo "You have the only IP that can access the instance now" + @. ve/bin/activate; ansible-playbook local.yml --check --flush-cache &> $(SHELL_OUTPUT) + +instance.ping: ve/bin/ansible-playbook ### Performs a ping to your instance. + @. ve/bin/activate; ansible -i $(INVENTORY) $(INSTANCE_NAME) -m ping \ + || (echo -e "\n${redbold}ERROR${red} Unable to ping instance!${normal}\n${bold}This might caused by one of the following reasons:${normal}\n\ + * The instance is not set up yet. To set up an instance run ${underline}${cyan}make instnace.setup${normal}.\n\ + * The instance was stopped. Check the status of your instance using ${underline}${cyan}make instance.describe.status${normal} and start it by running ${underline}${cyan}make instance.start${normal}.\n\ + * The instance might have been restricted under a previous IP of yours. To allow your current IP from accessing the instance run ${underline}${cyan}make instance.restrict${normal}."; \ + make error) + +image.create: instance.stop image.delete ### Creates an image from your instance on GCP. + @echo "Creating a new devstack image from your GCP instance... ${dim}($(IMAGE_NAME))${normal}" + @make NAME=$(IMAGE_NAME) image.create.command + @echo -e "${green}Your image has been successfully created${normal}" + +instance.restrict: instance.firewall.deny.refresh instance.firewall.allow.refresh ### Restricts the access to your instance to you only by creating the necessary rules. +ifeq ($(RESTRICT_INSTANCE),true) + $(eval MY_PUBLIC_IP := $(shell curl -s ifconfig.me)) + @echo -e "${green}The instance only communicates with your IP now${normal} ${dim}($(MY_PUBLIC_IP))${normal}" +else + @echo -e "\n${yellow}The instance accepts requests from any IP now.${normal} ${dim}(0.0.0.0/0)${normal}" +endif + @echo -e "If this is not the expected behavior, consider toggling the instance restriction setting ${bold}RESTRICT_INSTANCE${normal} in your env file." -instance.setup: clean instance.delete instance.create instance.restrict local.hosts.update local.ssh.config instance.deploy devstack.provision ## Setup a restricted instance for you on GCP contains a provisioned devstack. - @echo "Your instance created successfully" +instance.setup: clean instance.delete instance.create instance.restrict local.hosts.update local.ssh.config instance.deploy devstack.provision ### Setup a restricted instance for you on GCP contains a provisioned devstack. + @echo -e "${green}Your instance has been successfully created!${normal}" -instance.setup.image: clean instance.delete ## Setup a restricted instance for you on GCP (contains a devstack). +instance.setup.image: clean instance.delete ## Setup a restricted instance from an already created image for you on GCP (contains a devstack). @echo Setting up a new instance from your image... @gcloud compute instances create $(INSTANCE_NAME) \ --image=$(IMAGE_NAME) \ @@ -198,8 +109,8 @@ instance.setup.image: clean instance.delete ## Setup a restricted instance for y @make instance.restrict @make local.hosts.update @make local.ssh.config - @echo "Your instance created successfully from " $(IMAGE_NAME) - @echo "Run 'make devstack.run' to start the servers" + @echo -e "${green}Your instance has been successfully created!${normal} (From $(IMAGE_NAME))" + @echo -e "Run ${cyan}${underline}make instance.start${normal} and then ${cyan}${underline}make devstack.run${normal} to start devstack servers." instance.setup.image.master: clean instance.delete ## Setup a restricted instance from the master image. @echo Setting up a new instance from the master image... @@ -215,76 +126,5 @@ instance.setup.image.master: clean instance.delete ## Setup a restricted instanc @make instance.restrict @make local.hosts.update @make local.ssh.config - @echo "Your instance created successfully from " $(IMAGE_NAME) - @echo "Run 'make devstack.run' to start the servers" - -instance.run: ## SSH into or run commands on your instance. - - @ssh -tt devstack "$(command)" - -instance.ip: ## Gets the external IP of your instance. - @gcloud compute instances describe $(INSTANCE_NAME) \ - --zone=$(ZONE) \ - --project=$(PROJECT_ID) \ - --format='get(networkInterfaces[0].accessConfigs[0].natIP)' - -local.ssh.config: ve/bin/ansible-playbook - @echo Updating ~/.ssh/config file ... - $(eval IP_ADDRESS := $(shell make instance.ip)) - @. ve/bin/activate; ansible-playbook local.yml \ - --connection=local \ - -i '127.0.0.1,' \ - --tags ssh_config \ - -e "IP_ADDRESS=$(IP_ADDRESS) USER=$(USER_NAME) SSH_KEY=$(SSH_KEY)" > $(ANSIBLE_OUTPUT) - @ssh-add $(SSH_KEY) - -local.inventory.config: ve/bin/ansible-playbook - @echo Updating your inventory credentials ... - @. ve/bin/activate; ansible-playbook local.yml \ - --connection=local \ - -i '127.0.0.1,' \ - --tags inventory \ - -e "PROJECT_ID=$(PROJECT_ID) SERVICE_ACCOUNT_EMAIL=$(SERVICE_ACCOUNT_EMAIL) SERVICE_KEY_PATH=$(SERVICE_KEY_PATH)" > $(ANSIBLE_OUTPUT) - @ssh-add $(SSH_KEY) - -local.hosts.update: ve/bin/ansible-playbook ## Updates your hosts file by adding the necessary hosts to it. - @echo Updating /etc/hosts file ... - - $(eval IP_ADDRESS := $(shell make instance.ip)) - @. ve/bin/activate; sudo ansible-playbook --connection=local -i '127.0.0.1,' --tags hosts_update -e "IP_ADDRESS=$(IP_ADDRESS) TAHOE_HOST_NAME=$(TAHOE_HOST_NAME)" local.yml > $(ANSIBLE_OUTPUT) - -local.hosts.revert: ve/bin/ansible-playbook ## Updates your hosts file by removing the added hosts from it. - @echo Reverting changes made on /etc/hosts file ... - @echo Your local host sudo password might be required. - @. ve/bin/activate; sudo ansible-playbook --connection=local -i '127.0.0.1,' --tags hosts_revert local.yml > $(ANSIBLE_OUTPUT) - -git: ## Runs git commands against your remote devstack - @make instance.run command="(cd $(DEVSTACK_WORK_DIR)/$(repo) && git $(command))" - -devstack.make: ## Perfoms a make command on your instance. - @make instance.run command="(cd $(DEVSTACK_WORK_DIR)/devstack && make $(target))" - -devstack.run: ## Runs devstack servers. - @make instance.run command="cd $(DEVSTACK_WORK_DIR)/devstack && make HOST=$(TAHOE_HOST_NAME) tahoe.up.full" - -devstack.stop: ## Stops devstack servers. - @make devstack.make target=stop - -devstack.mount: ## Mounts the devstack from your instance onto your machine. - $(eval IP_ADDRESS := $(shell make instance.ip)) - - @mkdir -p $(MOUNT_DIR) - @echo "Mount directory created: " $(MOUNT_DIR) - @sshfs \ - -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=3,allow_other,defer_permissions,IdentityFile=$(SSH_KEY) \ - $(USER_NAME)@$(IP_ADDRESS):/home/$(USER_NAME)/work/tahoe-hawthorn \ - $(MOUNT_DIR) - -devstack.unmount: ## Releases the devstack mount from your machine. -ifeq ($(shell uname -s),Darwin) - @diskutil unmount force $(MOUNT_DIR) -else - @sudo unmount force $(MOUNT_DIR) -endif - - @rm -rf $(MOUNT_DIR) + @echo -e "${green}Your instance has been successfully created!${normal} (From $(IMAGE_FAMILY))" + @echo -e "Run ${cyan}${underline}make instance.start${normal} and then ${cyan}${underline}make devstack.run${normal} to start devstack servers." diff --git a/README.md b/README.md index 9e6e5fb..5e25c60 100644 --- a/README.md +++ b/README.md @@ -52,26 +52,32 @@ $ make instance.delete > The instance is secured behind a firewall that only identifies your IP. If your IP happened to change (after a network disconnection for example), run `make instance.restrict` to change your IP in the firewall rule. -## Mounting the devstack -To mount the devstack on your local machine -```shell -$ make devstack.mount -``` -Or to unmont: -```shell -$make devstack.unmount -``` + +## Development +There are so many ways you can choose from to interact with a remote code. However, we recommend two common methods that ensures security, real-time transfer, and immediate reflection: + +### SSHFS +We implicitly implemented this functionality within the toolkit. To use it all you have to do is to run `make devstack.mount` and then open your favorite text editor and start editing the files on the server from your machine. + +### Prefered IDEs +Some IDEs gives you the power to edit code on remote machines. [Visual Studio Code](https://code.visualstudio.com) for example, recently added [Code Remote Extensions](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack). With this extension, you'll be able to open any folder on your remote machine and take advantage of VS Code's full feature set. ## Creating an image To create an image from your running devstack, use the following ```shell -$ make instance.image.create +$ make image.create ``` To create an instance from your image ```shell $ make instance.setup.image ``` +## Environment files +We create a specific ignored .env file for you run `make environment.create`, to debug the final environment variables values you can run +```shell +$ make environment.debug +``` + ## Target help To check the targets documentation run ```shell diff --git a/local.yml b/local.yml index 42e7321..a5d68e4 100644 --- a/local.yml +++ b/local.yml @@ -2,32 +2,35 @@ - hosts: all tasks: - - name: Insert the Tahoe devstack hostnames + - name: Add mappings to /etc/hosts blockinfile: path: /etc/hosts block: | - # - # Those configurations are temporary and might removed - # or changed based on your devstack configurations. - - {{ IP_ADDRESS }} {{ TAHOE_HOST_NAME }} - {{ IP_ADDRESS }} edx.devstack.lms - {{ IP_ADDRESS }} red.localhost - {{ IP_ADDRESS }} blue.localhost - {{ IP_ADDRESS }} green.localhost + {{ IP_ADDRESS }} {{ item }} + marker: "# {mark} ANSIBLE MANAGED BLOCK {{ item }}" become: yes + with_items: "{{ EDX_HOST_NAMES.split(',') }}" tags: - hosts_update - - name: Remove Tahoe devstack hostnames + - name: Remove SSH configurations blockinfile: - path: "{{ item }}" + create: yes + path: ~/.ssh/config marker: "# {mark} ANSIBLE MANAGED BLOCK" block: "" become: yes - with_items: - - /etc/hosts - - ~/.ssh/config + tags: + - hosts_revert + + - name: Remove mappings from /etc/hosts + blockinfile: + path: /etc/hosts + marker: "# {mark} ANSIBLE MANAGED BLOCK {{ item }}" + create: yes + block: "" + become: yes + with_items: "{{ EDX_HOST_NAMES.split(',') }}" tags: - hosts_revert diff --git a/roles/devstack/tasks/main.yml b/roles/devstack/tasks/main.yml index 1e3ac20..c67410f 100644 --- a/roles/devstack/tasks/main.yml +++ b/roles/devstack/tasks/main.yml @@ -5,7 +5,6 @@ state: absent path: "{{ working_directory }}" tags: [ devstack ] - become: no - name: Create working directory file: diff --git a/roles/server/tasks/main.yml b/roles/server/tasks/main.yml index 2e4a717..f9bde8d 100644 --- a/roles/server/tasks/main.yml +++ b/roles/server/tasks/main.yml @@ -46,3 +46,10 @@ pip: name: pexpect become: yes + +- name: set inotify watchers + sysctl: + name=fs.inotify.max_user_watches + value=524288 + state=present + reload=yes diff --git a/server-vars.yml b/server-vars.yml index 72891b4..bc44576 100644 --- a/server-vars.yml +++ b/server-vars.yml @@ -1,12 +1,7 @@ -git_repo_url: https://github.com/appsembler/devstack.git -git_repo_branch: hawthorn - -working_directory: "~/workspace" devstack_directory: "{{ working_directory }}/devstack" ssh_keys_dir: "~/.ssh" - docker_group_name: "docker" user_home: "/home/{{ user }}" diff --git a/targets/devstack.mk b/targets/devstack.mk new file mode 100644 index 0000000..c40105f --- /dev/null +++ b/targets/devstack.mk @@ -0,0 +1,35 @@ +devstack.make: ## Perfoms a make command on your instance. + @make instance.run command="(cd $(DEVSTACK_WORK_DIR)/devstack && make $(target))" + +devstack.run: ### Runs devstack servers. + make instance.run command="cd $(DEVSTACK_WORK_DIR)/devstack && make $(DEVSTACK_RUN_COMMAND)" + @echo -e "${green}The devstack is up and running.${normal}" + +devstack.stop: devstack.unmount ### Stops and unmounts a devstack servers. + @make devstack.make target=stop + @echo -e "${green}Your devstack stopped successfully.${normal}" + +devstack.mount: ### Mounts the devstack from your instance onto your machine. + $(eval IP_ADDRESS := $(shell make instance.ip)) + + @mkdir -p $(MOUNT_DIR) + @echo -e "${bold}Mount directory created${normal}. ${dim}($(MOUNT_DIR))${normal}" + @sshfs \ + -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=3,allow_other,defer_permissions,IdentityFile=$(SSH_KEY) \ + $(USER_NAME)@$(IP_ADDRESS):$(DEVSTACK_WORK_DIR) \ + $(MOUNT_DIR) + @echo -e "${green}Workspace has been mounted successfully.${normal}" + +devstack.unmount: ## Releases the devstack mount from your machine. + $(eval UNMOUNT := $(shell make instance.ip)) + +ifeq ($(shell uname -s),Darwin) + $(eval UNMOUNT := diskutil) +else + $(eval UNMOUNT := sudo) +endif + + @($(UNMOUNT) unmount force $(MOUNT_DIR) && \ + rm -rf $(MOUNT_DIR) && \ + echo -e "${green}Workspace unmounted successfully.${normal}") \ + || echo -e "${yellow}No mount found${normal} (SKIPPING)" diff --git a/targets/firewall.mk b/targets/firewall.mk new file mode 100644 index 0000000..d4de0a9 --- /dev/null +++ b/targets/firewall.mk @@ -0,0 +1,58 @@ +instance.firewall.deny.delete: ## Deletes the GCP Firewall's rule that prevents accessing your instance by all ways. + @echo -e "Removing DENY firewall rule from gcp... ${dim}($(DENY_FIREWALL))${normal}" + @(gcloud compute firewall-rules delete $(DENY_FIREWALL) \ + --project=$(PROJECT_ID) \ + --verbosity $(VERBOSITY) \ + --quiet && echo -e "\n${green}DENY firewall has been successfully deleted!${normal}") || echo -e "${yellow}No previous deny firewall found. (SKIPPING)${normal}" + +instance.firewall.deny.create: ## Creates a GCP Firewall's rule to prevent all kind of access to your instance. +ifeq ($(RESTRICT_INSTANCE),true) + @echo -e "Creating DENY firewall rule in gcp... ${dim}($(DENY_FIREWALL))${normal}" + @(gcloud compute firewall-rules create $(DENY_FIREWALL) \ + --quiet \ + --verbosity $(VERBOSITY) \ + --action=deny \ + --direction=ingress \ + --rules=tcp \ + --source-ranges=0.0.0.0/0 \ + --priority=1000 \ + --target-tags=$(INSTANCE_TAG) \ + --project=$(PROJECT_ID) && echo -e "\n${green}DENY firewall has been successfully created!${normal}") || echo -e "${yellow}Firewall already exists. (SKIPPING)${normal}" +endif + +instance.firewall.allow.delete: ## Deletes the GCP Firewall's rule that allows your IP to access your instance. + @echo -e "Removing ALLOW firewall rule from gcp... ${dim}($(ALLOW_FIREWALL))${normal}" + @gcloud compute firewall-rules delete $(ALLOW_FIREWALL) \ + --project=$(PROJECT_ID) \ + --verbosity $(VERBOSITY) \ + --quiet \ + || echo -e "${yellow}No previous allow firewall found. (SKIPPING)${normal}" + +instance.firewall.allow.create: ## Creates a GCP Firewall's rule allowing your IP to access your instance. +ifeq ($(RESTRICT_INSTANCE),true) + $(eval MY_PUBLIC_IP := $(shell curl ifconfig.me)) +else + $(eval MY_PUBLIC_IP := 0.0.0.0/0) +endif + + @echo -e "Creating ALLOW firewall rule in gcp... ${dim}($(ALLOW_FIREWALL):$(MY_PUBLIC_IP))${normal}" + @(gcloud compute firewall-rules create $(ALLOW_FIREWALL) \ + --quiet \ + --verbosity $(VERBOSITY) \ + --action allow \ + --direction ingress \ + --rules tcp \ + --source-ranges $(MY_PUBLIC_IP) \ + --priority 50 \ + --target-tags=$(INSTANCE_TAG)\ + --project=$(PROJECT_ID) && echo -e "\n${green}ALLOW firewall has been successfully created!${normal}") || echo -e "${yellow}Firewall already exists. (SKIPPING)${normal}" + +instance.firewall.deny.refresh: instance.firewall.deny.delete instance.firewall.deny.create ## Refreshes the deny rule on GCP Firewall by deleting the old rule and creating a new one. +ifeq ($(RESTRICT_INSTANCE),true) + @echo -e "${green}Deny rule has been updated on the firewall.${normal}" +endif + +instance.firewall.allow.refresh: instance.firewall.allow.delete instance.firewall.allow.create ## Refreshes the allow rule on GCP Firewall by deleting the old rule and creating a new one. +ifeq ($(RESTRICT_INSTANCE),true) + @echo -e "${green}Allow rule has been updated on the firewall.${normal}" +endif diff --git a/targets/image.mk b/targets/image.mk new file mode 100644 index 0000000..15d9536 --- /dev/null +++ b/targets/image.mk @@ -0,0 +1,28 @@ +image.delete.command: + @echo Removing $(NAME) image from GCP... + @(gcloud compute images delete $(NAME) \ + --project=$(PROJECT_ID) \ + --verbosity $(VERBOSITY) \ + --quiet && \ + echo -e "${green}Image deleted successfully!${normal}") || echo -e "${yellow}Couldn't find the image on GCP. (SKIPPING)" + +image.delete: ## Deletes your image from GCP. + @make NAME=$(IMAGE_NAME) image.delete.command + +image.master.delete: + @make NAME=$(IMAGE_FAMILY) image.delete.command + +image.create.command: + @(gcloud beta compute images create $(NAME) \ + --source-disk=$(INSTANCE_NAME) \ + --source-disk-zone=$(ZONE) \ + --family=$(IMAGE_FAMILY) \ + --labels=user=$(INSTANCE_NAME) \ + --project=$(PROJECT_ID) \ + --verbosity $(VERBOSITY) \ + --quiet && \ + echo -e "${green}Image created successfully!${normal}") || echo -e "${yellow}Couldn't find the image on GCP. (SKIPPING)${normal}" + +image.master.create: instance.stop image.master.delete ## Creates a master image from your instance on GCP. + @echo -e "Creating a new master devstack image on GCP... ${dim}($(IMAGE_FAMILY))${normal}" + @make NAME=$(IMAGE_FAMILY) image.create.command diff --git a/targets/instnace.mk b/targets/instnace.mk new file mode 100644 index 0000000..994bd5b --- /dev/null +++ b/targets/instnace.mk @@ -0,0 +1,77 @@ + +instance.describe: ## Describes your virtual machine instance: + @gcloud compute instances describe $(INSTANCE_NAME) \ + --quiet \ + --zone=$(ZONE) \ + --verbosity $(VERBOSITY) \ + --project $(PROJECT_ID) \ + || echo -e "${yellow}No instance found${normal} (SKIPPING)" + +instance.describe.%: ## Shows a specific value of your virtual machine's metadata. Some helpful commands might be status, tags, disks, zone, and accessConfigs. + @gcloud compute instances describe $(INSTANCE_NAME) \ + --quiet \ + --zone=$(ZONE) \ + --verbosity $(VERBOSITY) \ + --project $(PROJECT_ID) \ + --format='value[]($*)' \ + || echo -e "${yellow}No instance found${normal} (SKIPPING)" + +instance.deploy: ve/bin/ansible-playbook ## Deploys your remote instance and prepare it for devstack provisioning. + @echo -e "Deploying your instance..." + @. ve/bin/activate; ansible-playbook devstack.yml \ + -i $(INVENTORY) \ + -e "instance_name=$(INSTANCE_NAME) working_directory=$(DEVSTACK_WORK_DIR) git_repo_url=$(DEVSTACK_REPO_URL) git_repo_branch=$(DEVSTACK_REPO_BRANCH)" &> $(SHELL_OUTPUT) + @echo -e "${green}Your virtual machine has been deployed successfully!${normal}" + @echo -e "Run ${cyan}${underline}make devstack.provision${normal} to start provisioning your devstack." + +devstack.provision: ## Provisions the devstack on your instance. + make instance.run command="cd $(DEVSTACK_WORK_DIR)/devstack/ && make dev.provision" + @echo -e "${green}The devstack has been provisioned successfully!${normal}" + @echo -e "Run ${cyan}${underline}make devstack.run${normal} to start devstack servers." + +instance.delete: local.hosts.revert instance.firewall.deny.delete instance.firewall.allow.delete ### Deletes your instance from GCP. + @echo -e "Deleting your virtual machine from GCP... ($(INSTANCE_NAME))" + @(gcloud compute instances delete $(INSTANCE_NAME) \ + --quiet \ + --zone=$(ZONE) \ + --verbosity $(VERBOSITY) \ + --project $(PROJECT_ID) && echo -e "${green}Your virtual machine has been deleted successfully!${normal}") \ + || echo 'No previous instance found' + +instance.start: ## Starts your stopped instance on GCP. + @echo -e "Starting your virtual machine on GCP... ($(INSTANCE_NAME))" + @gcloud compute instances start $(INSTANCE_NAME) \ + --zone=$(ZONE) \ + --project $(PROJECT_ID) + @make local.hosts.update + @make local.ssh.config + @echo -e "${green}Your virtual machine has been started successfully!${normal}" + +instance.stop: local.hosts.revert ## Stops your instance on GCP, but doesn't delete it. + @echo -e "Stopping your virtual machine on GCP... ($(INSTANCE_NAME))" + @gcloud compute instances stop $(INSTANCE_NAME) \ + --zone=$(ZONE) \ + --project $(PROJECT_ID) + @echo -e "${green}Your virtual machine has been stopped successfully!${normal}" + +instance.create: ## Creates an empty instance for you on GCP. + @echo -e "Creating your virtual machine on GCP... ($(INSTANCE_NAME))" + @gcloud compute instances create $(INSTANCE_NAME) \ + --image-family=ubuntu-1804-lts \ + --image-project=gce-uefi-images \ + --boot-disk-size=$(DISK_SIZE) \ + --machine-type=$(MACHINE_TYPE) \ + --tags=devstack,http-server,$(INSTANCE_TAG) \ + --zone=$(ZONE) \ + --verbosity $(VERBOSITY) \ + --project=$(PROJECT_ID) + @echo -e "${green}Your virtual machine has been successfully created!${normal}" + +instance.run: ## SSH into or run commands on your instance. + @ssh -tt devstack "$(command)" + +instance.ip: ## Gets the external IP of your instance. + @gcloud compute instances describe $(INSTANCE_NAME) \ + --zone=$(ZONE) \ + --project=$(PROJECT_ID) \ + --format='get(networkInterfaces[0].accessConfigs[0].natIP)' diff --git a/targets/local.mk b/targets/local.mk new file mode 100644 index 0000000..3bba41b --- /dev/null +++ b/targets/local.mk @@ -0,0 +1,79 @@ + +environment.debug: ## Prints the values of the environemnt variables to be used in the make command as define in .env.* files. + @echo -e "${cyan}ALLOW_FIREWALL${normal} $(ALLOW_FIREWALL)" + @echo -e "${cyan}DENY_FIREWALL${normal} $(DENY_FIREWALL)" + @echo -e "${cyan}DEVSTACK_REPO_BRANCH${normal} $(DEVSTACK_REPO_BRANCH)" + @echo -e "${cyan}DEVSTACK_REPO_URL${normal} $(DEVSTACK_REPO_URL)" + @echo -e "${cyan}DEVSTACK_RUN_COMMAND${normal} $(DEVSTACK_RUN_COMMAND)" + @echo -e "${cyan}DEVSTACK_WORK_DIR${normal} $(DEVSTACK_WORK_DIR)" + @echo -e "${cyan}DISK_SIZE${normal} $(DISK_SIZE)" + @echo -e "${cyan}EDX_HOST_NAMES{normal} $(EDX_HOST_NAMES)" + @echo -e "${cyan}HOST_NAMES{normal} $(HOST_NAMES)" + @echo -e "${cyan}HOSTS_FILE${normal} $(HOSTS_FILE)" + @echo -e "${cyan}IMAGE_FAMILY${normal} $(IMAGE_FAMILY)" + @echo -e "${cyan}IMAGE_NAME${normal} $(IMAGE_NAME)" + @echo -e "${cyan}INSTANCE_NAME${normal} $(INSTANCE_NAME)" + @echo -e "${cyan}INSTANCE_TAG${normal} $(INSTANCE_TAG)" + @echo -e "${cyan}INVENTORY${normal} $(INVENTORY)" + @echo -e "${cyan}MACHINE_TYPE${normal} $(MACHINE_TYPE)" + @echo -e "${cyan}MOUNT_DIR${normal} $(MOUNT_DIR)" + @echo -e "${cyan}PROJECT_ID${normal} $(PROJECT_ID)" + @echo -e "${cyan}RESTRICT_INSTANCE${normal} $(RESTRICT_INSTANCE)" + @echo -e "${cyan}SERVICE_ACCOUNT_EMAIL${normal} $(SERVICE_ACCOUNT_EMAIL)" + @echo -e "${cyan}SERVICE_KEY_PATH${normal} $(SERVICE_KEY_PATH)" + @echo -e "${cyan}SHELL_OUTPUT${normal} $(SHELL_OUTPUT)" + @echo -e "${cyan}SSH_KEY${normal} $(SSH_KEY)" + @echo -e "${cyan}TMP_DIR${normal} $(TMP_DIR)" + @echo -e "${cyan}USER_NAME${normal} $(USER_NAME)" + @echo -e "${cyan}VERBOSITY${normal} $(VERBOSITY)" + @echo -e "${cyan}ZONE${normal} $(ZONE)" + +local.ssh.config: ve/bin/ansible-playbook + @echo -e "Updating necessary records in SSH related files... ${dim}[~/.ssh/config, ~/.ssh/knwon_hosts]${normal}" + $(eval IP_ADDRESS := $(shell make instance.ip)) + @. ve/bin/activate; ansible-playbook local.yml \ + --connection=local \ + -i '127.0.0.1,' \ + --tags ssh_config \ + -e "IP_ADDRESS=$(IP_ADDRESS) USER=$(USER_NAME) SSH_KEY=$(SSH_KEY)" > $(SHELL_OUTPUT) || (echo -e "${redbold}ERROR configuring SSH connection in your machine.${normal}" && make error && exit 1) + @ssh-add $(SSH_KEY) + @echo -e "${green}SSH connection between your machine and the instnace has been configured successfully!${normal}" + +local.inventory.config: ve/bin/ansible-playbook + @echo -e "Updating your inventory credentials... ${dim}(dynamic-inventory/gce.ini)${normal}" + @. ve/bin/activate; ansible-playbook local.yml \ + --connection=local \ + -i '127.0.0.1,' \ + --tags inventory \ + -e "PROJECT_ID=$(PROJECT_ID) SERVICE_ACCOUNT_EMAIL=$(SERVICE_ACCOUNT_EMAIL) SERVICE_KEY_PATH=$(SERVICE_KEY_PATH)" > $(SHELL_OUTPUT) || (echo -e "${redbold}ERROR configuring your inventory.${normal}" && make error && exit 1) + @ssh-add $(SSH_KEY) + @echo -e "${green}Your inventory has been configured successfully!${normal}" + +local.hosts.update: ve/bin/ansible-playbook ## Updates your hosts file by adding the necessary hosts to it. + @echo -e "Updating your hosts records... ${dim}(/etc/hosts)${normal}" + + $(eval IP_ADDRESS := $(shell make instance.ip)) + @make sudocheck + @. ve/bin/activate; sudo ansible-playbook \ + --connection=local \ + -i '127.0.0.1,' \ + --tags hosts_update \ + -e "IP_ADDRESS=$(IP_ADDRESS) EDX_HOST_NAMES=$(EDX_HOST_NAMES)" local.yml > $(SHELL_OUTPUT) || (echo -e "${redbold}ERROR configuring hosts records.${normal}" && make error && exit 1) + @echo -e "${green}Your hosts have been configured successfully!${normal}" + +local.hosts.revert: ve/bin/ansible-playbook ## Updates your hosts file by removing the added hosts from it. + @echo -e "Reverting made local changes... ${dim}[/etc/hosts, ~/.ssh/config]${normal}" + @make sudocheck + @. ve/bin/activate; sudo ansible-playbook \ + --connection=local \ + -i '127.0.0.1,' \ + -e "EDX_HOST_NAMES=$(EDX_HOST_NAMES)" \ + --tags hosts_revert local.yml > $(SHELL_OUTPUT) || (echo -e "${redbold}ERROR reverting local changes.${normal}" && make error && exit 1) + @echo -e "${green}Your local changes have been reverted successfully!${normal}" + +CXXARCH:=$(shell $(CXX) -dumpmachine | grep -i 'x86_64') + +sudocheck: + @if ! sudo -n true 2>/dev/null ; then \ + echo -e "${yellow}Please enter your sudo password...${normal}"; \ + fi