From 92cd1fef0d456f4ef158e46831beaf5394ca340a Mon Sep 17 00:00:00 2001 From: Chris Plock Date: Mon, 13 Nov 2017 16:52:20 -0800 Subject: [PATCH] v2plugin set forward mode when netmaster up (#1058) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * v2plugin set forward mode when netmaster up Docker expects the netplugin socket to be available within 10 seconds before it fails enabling (or installing) the v2plugin. Due to #1043, netplugin is blocking waiting for the forward mode to be set, which is done by netctl calling netmaster, but netmaster is not started until the plugin is activating. Instead of backgrounding the plugin install/enabling then letting ansible set the forward mode, do it in the plugin script to avoid ansible's unpredictable round trip delays. v2plugin’s startcontiv.sh errors when fwd_mode not set Signed-off-by: Chris Plock --- Makefile | 1 + install/v2plugin/README.md | 20 ++++++++++--------- install/v2plugin/config.template | 8 ++++++++ install/v2plugin/startcontiv.sh | 34 ++++++++++++++++++++++++++++---- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b6ebf967f..2fc6862b8 100755 --- a/Makefile +++ b/Makefile @@ -367,6 +367,7 @@ demo-v2plugin: ssh-build make-on-node1-dep v2plugin-install host-plugin-release: tar host-pluginfs-create host-pluginfs-unpack host-plugin-create @echo dev: pushing ${CONTIV_V2PLUGIN_NAME} to docker hub @echo dev: need docker login with user in contiv org + @echo "dev: docker login --username " docker plugin push ${CONTIV_V2PLUGIN_NAME} # unarchive versioned binaries to bin, usually as a helper for other targets diff --git a/install/v2plugin/README.md b/install/v2plugin/README.md index 568d30a47..f7c481d6a 100644 --- a/install/v2plugin/README.md +++ b/install/v2plugin/README.md @@ -28,16 +28,18 @@ plugin_role : In 'master' role, plugin runs netmaster : master listen_url : Netmaster url to listen http requests on : ":9999" control_url : Netmaster url for control messages : ":9999" dbg_flag : To enable debug mode, set to '-debug' : "" +fwd_mode : Forwarding mode : "" + ``` ### docker store Docker certified contiv plugin is avaliable on [Docker Store](https://store.docker.com/plugins/803eecee-0780-401a-a454-e9523ccf86b3?tab=description). ``` -docker plugin install store/contiv/v2plugin: iflist= +docker plugin install store/contiv/v2plugin: iflist= fwd_mode= ``` ### docker hub Developer release of v2plugin from contiv repo is also pushed to docker hub ``` -docker plugin install contiv/v2plugin: iflist= +docker plugin install contiv/v2plugin: iflist= fwd_mode= ``` ### vagrant dev/demo setup To create a plugin from [contiv repo](https://github.com/contiv/netplugin), enable v2plugin and run docker in swarm-mode, use the Makefile target demo-v2plugin @@ -48,7 +50,7 @@ make demo-v2plugin ## Contiv plugin-roles Contiv plugin runs both netplugin and netmaster by default. Contiv v2plugin can be run with only netplugin by setting the plugin_role to worker. ``` -docker plugin install contiv/v2plugin: iflist= plugin_role=worker +docker plugin install contiv/v2plugin: iflist= plugin_role=worker fwd_mode= ``` ## Contiv plugin swarm-mode workflow (recommended and default for v2plugin) @@ -68,11 +70,11 @@ docker plugin install contiv/v2plugin: iflist= plugin_role=master iflist= + docker plugin install contiv/v2plugin: plugin_role=master iflist= fwd_mode= ( allow/grant the install permissions when prompted ) # on worker nodes, install plugin with 'worker' role - docker plugin install contiv/v2plugin: plugin_role=worker iflist= + docker plugin install contiv/v2plugin: plugin_role=worker iflist= fwd_mode= # to see if the plugin is installed and enabled docker plugin ls @@ -81,7 +83,7 @@ docker plugin install contiv/v2plugin: iflist= ctrl_ip=192.168.2.10 control_url=192.168.2.10:9999 iflist=eth2,eth3 + docker plugin install contiv/v2plugin: ctrl_ip=192.168.2.10 control_url=192.168.2.10:9999 iflist=eth2,eth3 fwd_mode=bridge ``` 4. Debug logs ``` @@ -128,15 +130,15 @@ docker plugin install contiv/v2plugin: iflist= plugin-mode=docker iflist= + docker plugin install contiv/v2plugin: plugin-mode=docker iflist= fwd_mode= ( allow/grant the install permissions when prompted ) # on node where netmaster needs to run, install plugin with 'master' role - docker plugin install contiv/v2plugin: plugin_role=master iflist= + docker plugin install contiv/v2plugin: plugin_role=master iflist= fwd_mode= ( allow/grant the install permissions when prompted ) # on all other nodes, install plugin with 'worker' role - docker plugin install contiv/v2plugin: plugin_role=worker iflist= + docker plugin install contiv/v2plugin: plugin_role=worker iflist= fwd_mode= # to see if the plugin is installed properly and enabled docker plugin ls diff --git a/install/v2plugin/config.template b/install/v2plugin/config.template index 652ad975a..7d13350a6 100644 --- a/install/v2plugin/config.template +++ b/install/v2plugin/config.template @@ -103,6 +103,14 @@ ], ## Do not change the default value, this will be replaced with $CONTIV_V2PLUGIN_NAME "Value": "__CONTIV_V2PLUGIN_NAME__" + }, + { + "Description": "Forwarding mode for netplugin", + "Name": "fwd_mode", + "Settable": [ + "value" + ], + "Value": "" } ], "mounts": [ diff --git a/install/v2plugin/startcontiv.sh b/install/v2plugin/startcontiv.sh index 43487df84..fdc101316 100755 --- a/install/v2plugin/startcontiv.sh +++ b/install/v2plugin/startcontiv.sh @@ -3,11 +3,19 @@ ### Pre-requisite on the host # run a cluster store like etcd or consul +set -e + if [ $log_dir == "" ]; then log_dir="/var/log/contiv" fi BOOTUP_LOGFILE="$log_dir/plugin_bootup.log" +# Redirect stdout and stdin to BOOTUP_LOGFILE +exec 1<&- # Close stdout +exec 2<&- # Close stderr +exec 1<>$BOOTUP_LOGFILE # stdout read and write to logfile instead of console +exec 2>&1 # redirect stderr to where stdout is (logfile) + mkdir -p $log_dir mkdir -p /var/run/openvswitch mkdir -p /etc/openvswitch @@ -33,8 +41,6 @@ if [ $vxlan_port != "4789" ]; then vxlan_port_cfg="-vxlan-port=$vxlan_port" fi -set -e - echo "Loading OVS" >> $BOOTUP_LOGFILE (modprobe openvswitch) || (echo "Load ovs FAILED!!! " >> $BOOTUP_LOGFILE) @@ -50,7 +56,7 @@ echo " Starting OVSBD server " >> $BOOTUP_LOGFILE ovsdb-server --remote=punix:/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --log-file=$log_dir/ovs-db.log -vsyslog:dbg -vfile:dbg --pidfile --detach /etc/openvswitch/conf.db >> $BOOTUP_LOGFILE echo " Starting ovs-vswitchd " >> $BOOTUP_LOGFILE ovs-vswitchd -v --pidfile --detach --log-file=$log_dir/ovs-vswitchd.log -vconsole:err -vsyslog:info -vfile:info & -ovs-vsctl set-manager tcp:127.0.0.1:6640 +ovs-vsctl set-manager tcp:127.0.0.1:6640 ovs-vsctl set-manager ptcp:6640 echo "Started OVS, logs in $log_dir" >> $BOOTUP_LOGFILE @@ -68,6 +74,10 @@ while true ; do done & if [ $plugin_role == "master" ]; then + if [ -z "$fwd_mode" ]; then + echo "fwd_mode is not set, plugin cannot be enabled" + exit 1 + fi echo "Starting Netmaster " >> $BOOTUP_LOGFILE while true ; do echo "/netmaster $dbg_flag -plugin-name=$plugin_name -cluster-mode=$plugin_mode -cluster-store=$cluster_store $listen_url_cfg $control_url_cfg" >> $BOOTUP_LOGFILE @@ -77,9 +87,25 @@ if [ $plugin_role == "master" ]; then sleep 5 echo "Restarting Netmaster " >> $BOOTUP_LOGFILE done & + + set -e + echo "Waiting for netmaster to be ready for connections" + # wait till netmaster starts to listen + for i in $(seq 1 10); do + [ "$(curl -s -o /dev/null -w '%{http_code}' $control_url)" != "000" ] \ + && break + sleep 1 + done + if [ "$i" -ge "10" ]; then + echo "netmaster port not open (needed to set forwarding mode), plugin failed" + exit 1 + fi + sleep 1 + echo "Netmaster ready for connections, setting forward mode to $fwd_mode" + /netctl --netmaster http://$control_url global set --fwd-mode "$fwd_mode" + echo "Forward mode is set" else echo "Not starting netmaster as plugin role is" $plugin_role >> $BOOTUP_LOGFILE fi while true; do sleep 1; done -