Commit beeccafa authored by Andy Cobaugh's avatar Andy Cobaugh

Merge branch 'feature/ci_wait' into 'develop'

Add ci wait subcommand

See merge request !10
parents f8c8a884 0fb48616
Pipeline #96402 passed with stages
in 1 minute and 8 seconds
package cmd
import (
"bytes"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"os"
"text/template"
"time"
"github.com/apex/log"
"github.com/apex/log/handlers/cli"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// buildCmd represents the build command
var ciWaitCmd = &cobra.Command{
Use: "wait",
Short: "Wait for a deployment to be deployed by watching the live version endpoint",
RunE: runCiWaitCmd,
PreRunE: func(cmd *cobra.Command, args []string) error {
return viper.BindPFlags(cmd.PersistentFlags())
},
}
var ciWaitCommitFields = []string{"scm-commit-id", "Commit", "commit"}
var ciWaitVersionFields = []string{"Version", "version", "commit"}
var ciWaitURLsDefault = []string{
"https://{{.Env}}-{{.Project}}.qa.k8s.psu.edu/{{.Project}}-web/resources/version",
"https://{{.Env}}-{{.Project}}.qa.k8s.psu.edu/{{.Project}}/resources/version",
"https://{{.Env}}-{{.Project}}.qa.k8s.psu.edu/resources/version",
"https://{{.Env}}-{{.Project}}.qa.k8s.psu.edu/version",
}
type ciWaitBaseURLData struct {
Project string
Env string
}
func init() {
ciWaitCmd.PersistentFlags().StringSliceP("urls", "u", ciWaitURLsDefault, "List of URL templates to check in order to find version endpoint")
ciWaitCmd.PersistentFlags().Int("url-max-tries", 10, "Max tries to locate the version endpoint")
ciWaitCmd.PersistentFlags().Duration("url-delay", 10*time.Second, "Time to wait between attempts to locate version endpoint")
ciWaitCmd.PersistentFlags().Int("update-max-tries", 30, "Max tries to check version endpoint for the update")
ciWaitCmd.PersistentFlags().Duration("update-delay", 10*time.Second, "Time to wait between checking version endpoint for update")
ciWaitCmd.PersistentFlags().Duration("timeout", 2*time.Second, "Time to wait for every HTTP request")
ciWaitCmd.PersistentFlags().Bool("wait-for-rollout", true, "Wait for the rollout to finish by repeatedly checking the version endpoint every [rollout-delay]s until we see the new version [rollout-count] times in a row")
ciWaitCmd.PersistentFlags().Duration("rollout-delay", 15*time.Second, "Time to wait between tries when waiting for rollout")
ciWaitCmd.PersistentFlags().Int("rollout-count", 5, "Number of successive requests that must return the new version before the rollout is considered done")
ciCmd.AddCommand(ciWaitCmd)
}
func runCiWaitCmd(cmd *cobra.Command, args []string) error {
start := time.Now()
data := ciWaitBaseURLData{
Project: os.Getenv("CI_PROJECT_NAME"),
Env: environmentSuffix,
}
newCommit := os.Getenv("CI_COMMIT_SHA")
newVersion := os.Getenv("CI_COMMIT_TAG")
var url string
var initialBody []byte
var newBody []byte
hc := &http.Client{Timeout: viper.GetDuration("timeout")}
log.WithFields(log.Fields{
"commit": newCommit,
"version": newVersion,
"project": data.Project,
"env": data.Env,
}).Info("Waiting for new version to be available")
log.WithFields(log.Fields{
"maxtries": viper.GetInt("url-max-tries"),
"delay": viper.GetDuration("url-delay").String(),
}).Info("Looking for version endpoint")
// loop over possible urls
cli.Default.Padding = 2 * InitialPadding
for i := 1; i <= viper.GetInt("url-max-tries"); i++ {
for _, u := range viper.GetStringSlice("urls") {
tmpl, err := template.New("URL").Parse(u)
if err != nil {
log.Fatalf("Could not parse URL template '%s': %s", u, err)
}
var urlBuf bytes.Buffer
err = tmpl.Execute(&urlBuf, data)
if err != nil {
log.Fatalf("Could not execute URL template: %s", err)
}
// check url for 200 response
log.WithField("try", i).Infof("Checking URL: %s", urlBuf.String())
resp, err := hc.Get(urlBuf.String())
if err != nil {
log.WithError(err).Error("Error fetching URL")
continue
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
log.WithFields(log.Fields{"code": resp.StatusCode}).Info("Found version endpoint")
url = urlBuf.String()
initialBody, err = ioutil.ReadAll(resp.Body)
if err != nil {
log.WithError(err).Error("Could not read body")
}
break
} else {
log.WithFields(log.Fields{"code": resp.StatusCode}).Warn("No version endpoint here")
}
}
if url == "" {
log.Warnf("Sleeping for %s ...", viper.GetDuration("url-delay").String())
time.Sleep(viper.GetDuration("url-delay"))
} else {
break
}
}
cli.Default.Padding = InitialPadding
if url == "" {
log.Fatalf("Could not locate version endpoint after %s", time.Since(start).Truncate(time.Millisecond))
}
log.WithFields(log.Fields{
"project": data.Project,
"env": data.Env,
"url": url,
"maxtries": viper.GetInt("update-max-tries"),
"delay": viper.GetDuration("update-delay").String(),
}).Infof("Waiting for version update")
cli.Default.Padding = 2 * InitialPadding
// wait for version to update
for i := 1; i <= viper.GetInt("update-max-tries"); i++ {
log.WithField("try", i).Infof("Checking URL: %s", url)
// check version url and compare
resp, err := hc.Get(url)
if err != nil {
log.WithError(err).WithField("url", url).Error("Error fetching URL")
continue
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
log.WithFields(log.Fields{"code": resp.StatusCode}).Info("Received response")
newBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.WithError(err).Error("Could not read body")
continue
}
if ciWaitVersionCompare(initialBody, newBody, newCommit, newVersion) {
log.Info("New version is live")
seen := 1
// optionally wait for the rollout
if viper.GetBool("wait-for-rollout") {
cli.Default.Padding = InitialPadding
log.WithFields(log.Fields{"seen": seen, "delay": viper.GetDuration("rollout-delay"), "count": viper.GetInt("rollout-count")}).Info("Waiting for rollout")
cli.Default.Padding = 2 * InitialPadding
for seen < viper.GetInt("rollout-count") {
time.Sleep(viper.GetDuration("rollout-delay"))
resp, err := hc.Get(url)
if err != nil {
log.WithError(err).WithField("url", url).Error("Error fetching URL")
continue
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK && ciWaitVersionCompare(initialBody, newBody, newCommit, newVersion) {
seen++
log.WithField("seen", seen).Info("NEW")
continue
}
// we consider non-200 statuses and a response with the old version to be failures
// reset seen back to 0 and continue
seen = 0
log.WithField("seen", seen).Warn("OLD")
}
}
cli.Default.Padding = InitialPadding
log.Infof("Finished after %s", time.Since(start).Truncate(time.Millisecond))
return nil
} else {
log.Warn("No update to version endpoint")
}
}
log.WithFields(log.Fields{"code": resp.StatusCode}).Warn("No OK response")
log.Warnf("Sleeping for %s ...", viper.GetDuration("update-delay").String())
time.Sleep(viper.GetDuration("update-delay"))
}
cli.Default.Padding = InitialPadding
log.WithFields(log.Fields{
"url": url,
"old": initialBody,
"new": newBody,
"commit": newCommit,
"version": newVersion,
}).Errorf("Did not observe an update to the version endpoint after %s", time.Since(start).Truncate(time.Millisecond))
return errors.New("Did not observe an update to the version endpoint")
}
// ciWaitVersionCompare returns true if:
// 1) the version element contained in newBody matches vesion
// or 2) the commit element contained in newBody matches commit
// or 3) if all else fails, newBody is different from oldBody
func ciWaitVersionCompare(initialBody []byte, newBody []byte, commit string, version string) bool {
var nb map[string]interface{}
err := json.Unmarshal(newBody, &nb)
if err != nil {
// unable to unmarshal newBody, do a straight string comparison
return string(initialBody) == string(newBody)
}
// look for long commit, comparing the first len(commit) characters
if commit != "" {
for _, k := range ciWaitCommitFields {
if v := nb[k]; v != "" {
l := len(v.(string))
if len(commit) >= l {
return v == commit[:l]
}
}
}
}
// look for version
for _, k := range ciWaitVersionFields {
if v := nb[k]; v != "" {
return v == version
}
}
// all else fails, do a straight string comparison
return string(initialBody) == string(newBody)
}
......@@ -19,6 +19,8 @@ import (
"os"
"strings"
"github.com/apex/log"
"github.com/apex/log/handlers/cli"
"github.com/fatih/color"
homedir "github.com/mitchellh/go-homedir"
"github.com/spf13/cobra"
......@@ -31,6 +33,8 @@ const (
var cfgFile string
const InitialPadding = 2
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "devtool",
......@@ -57,6 +61,8 @@ func Execute() {
}
func init() {
log.SetHandler(cli.Default)
cli.Default.Padding = InitialPadding
cobra.OnInitialize(initConfig)
// Here you will define your flags and configuration settings.
......
......@@ -5,6 +5,7 @@ go 1.12
require (
git.psu.edu/swe-golang/buildversion v0.2.0
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
github.com/apex/log v1.1.2
github.com/fatih/color v1.7.0
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/goodhosts/hostsfile v0.0.1
......@@ -13,7 +14,6 @@ require (
github.com/magiconair/properties v1.8.1 // indirect
github.com/manifoldco/promptui v0.3.2
github.com/markbates/pkger v0.15.0
github.com/mattn/go-colorable v0.1.2 // indirect
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.2.2 // indirect
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment