// doReconnect attempts a reconnection.  It assumes that reconnectChan
// and reconnectErrPtr are the same ones in c, but are passed in to
// avoid having to take the mutex at the beginning of the method.
func (c *Connection) doReconnect(ctx context.Context,
	reconnectChan chan struct{}, reconnectErrPtr *error) {
	// retry w/exponential backoff
	backoff.RetryNotify(func() error {
		// try to connect
		err := c.connect(ctx)
		select {
		case <-ctx.Done():
			// context was canceled by Shutdown() or a user action
			*reconnectErrPtr = ctx.Err()
			// short-circuit Retry
			return nil
		default:
		}
		if dontRetryOnConnect(err) {
			// A fatal error happened.
			*reconnectErrPtr = err
			// short-circuit Retry
			return nil
		}
		return err
	}, backoff.NewExponentialBackOff(),
		// give the caller a chance to log any other error or adjust state
		c.handler.OnConnectError)

	// close the reconnect channel to signal we're connected.
	c.mutex.Lock()
	defer c.mutex.Unlock()
	close(reconnectChan)
	c.reconnectChan = nil
	c.cancelFunc = nil
	c.reconnectErrPtr = nil
}

// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints []types.Constraint) error {
	config := api.DefaultConfig()
	config.Address = provider.Endpoint
	client, err := api.NewClient(config)
	if err != nil {
		return err
	}
	provider.client = client
	provider.Constraints = append(provider.Constraints, constraints...)

	pool.Go(func(stop chan bool) {
		notify := func(err error, time time.Duration) {
			log.Errorf("Consul connection error %+v, retrying in %s", err, time)
		}
		worker := func() error {
			return provider.watch(configurationChan, stop)
		}
		err := backoff.RetryNotify(worker, backoff.NewExponentialBackOff(), notify)
		if err != nil {
			log.Fatalf("Cannot connect to consul server %+v", err)
		}
	})

	return err
}

func portUpdate(c *config.Config, ctx context.Context) error {
	ip, er := getIP(c.OpenVPN.Tun, c.Timeout.Duration, ctx)
	if er != nil || ctx.Err() != nil {
		return er
	}
	logger.Infof("New bind ip: (%s) %s", c.OpenVPN.Tun, ip)

	port, er := getPort(ip, c.PIA.User, c.PIA.Pass, c.PIA.ClientID, c.Timeout.Duration, ctx)
	if er != nil || ctx.Err() != nil {
		return er
	}

	logger.Infof("New peer port: %d", port)
	notify := func(e error, w time.Duration) {
		logger.Debugf("Failed to update transmission port: %v", er)
	}
	operation := func() error {
		select {
		default:
			return transmission.
				NewRawClient(c.Transmission.URL.String(), c.Transmission.User, c.Transmission.Pass).
				UpdatePort(port)
		case <-ctx.Done():
			return nil
		}
	}
	b := backoff.NewExponentialBackOff()
	b.MaxElapsedTime = c.Timeout.Duration
	return backoff.RetryNotify(operation, b, notify)
}

func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errChan chan<- error) {
	var body string
	var errs []error
	var resp *http.Response
	f := func() (err error) {
		//  resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
		resp, body, errs = gorequest.New().Get(url).End()
		if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
			return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", errs, url, resp)
		}
		return nil
	}
	notify := func(err error, t time.Duration) {
		log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
	}
	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
	if err != nil {
		errChan <- fmt.Errorf("HTTP Error %s", err)
	}
	cveDetail := cve.CveDetail{}
	if err := json.Unmarshal([]byte(body), &cveDetail); err != nil {
		errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
	}
	resChan <- response{
		key,
		cveDetail,
	}
}

func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix string, stop chan bool) {
	operation := func() error {
		events, err := provider.kvclient.WatchTree(provider.Prefix, make(chan struct{}) /* stop chan */)
		if err != nil {
			log.Errorf("Failed to WatchTree %s", err)
			return err
		}
		for {
			select {
			case <-stop:
				return nil
			case _, ok := <-events:
				if !ok {
					return errors.New("watchtree channel closed")
				}
				configuration := provider.loadConfig()
				if configuration != nil {
					configurationChan <- types.ConfigMessage{
						ProviderName:  string(provider.storeType),
						Configuration: configuration,
					}
				}
			}
		}
	}

	notify := func(err error, time time.Duration) {
		log.Errorf("KV connection error %+v, retrying in %s", err, time)
	}
	err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
	if err != nil {
		log.Fatalf("Cannot connect to KV server %+v", err)
	}
}

func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) {

	if client = tryAgentConnect(c); client != nil {
		return client, nil
	}

	var auths = []ssh.AuthMethod{}
	if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil {
		logrus.Fatalf("Failed to add keyAuth. %[email protected]%s:%s err: %s",
			c.User, c.Host, c.Port, err)
	}

	if c.Password != "" {
		auths = append(auths, ssh.Password(c.Password))
	}

	// http://blog.ralch.com/tutorial/golang-ssh-connection/
	config := &ssh.ClientConfig{
		User: c.User,
		Auth: auths,
	}

	notifyFunc := func(e error, t time.Duration) {
		logrus.Warnf("Failed to ssh %[email protected]%s:%s err: %s, Retrying in %s...",
			c.User, c.Host, c.Port, e, t)
	}
	err = backoff.RetryNotify(func() error {
		if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil {
			return err
		}
		return nil
	}, backoff.NewExponentialBackOff(), notifyFunc)

	return
}

func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cve.CveDetail, error) {
	var body string
	var errs []error
	var resp *http.Response
	f := func() (err error) {
		req := gorequest.New().SetDebug(config.Conf.Debug).Post(url)
		for key := range query {
			req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json")
		}
		resp, body, errs = req.End()
		if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
			return fmt.Errorf("HTTP POST error: %v, url: %s, resp: %v", errs, url, resp)
		}
		return nil
	}
	notify := func(err error, t time.Duration) {
		log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err)
	}
	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
	if err != nil {
		return []cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err)
	}

	cveDetails := []cve.CveDetail{}
	if err := json.Unmarshal([]byte(body), &cveDetails); err != nil {
		return []cve.CveDetail{},
			fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
	}
	return cveDetails, nil
}

func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) {
	if client = tryAgentConnect(c); client != nil {
		return client, nil
	}

	var auths = []ssh.AuthMethod{}
	if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil {
		return nil, err
	}

	// http://blog.ralch.com/tutorial/golang-ssh-connection/
	config := &ssh.ClientConfig{
		User: c.User,
		Auth: auths,
	}

	notifyFunc := func(e error, t time.Duration) {
		logger := getSSHLogger()
		logger.Debugf("Failed to Dial to %s, err: %s, Retrying in %s...",
			c.ServerName, e, t)
	}
	err = backoff.RetryNotify(func() error {
		if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil {
			return err
		}
		return nil
	}, backoff.NewExponentialBackOff(), notifyFunc)

	return
}

func (c *consulCoordinator) Start(addr net.Addr, errCh chan error) error {
	if addr == nil {
		addr = &net.TCPAddr{}
	}
	c.addr = addr
	session := c.client.Session()
	// set session to delete our keys on invalidation
	sessionOptions := &api.SessionEntry{
		Behavior:  api.SessionBehaviorDelete,
		LockDelay: c.config.LockDelay,
		TTL:       c.config.TTL,
	}
	var sessionID string
	var err error
	err = backoff.RetryNotify(func() error {
		sessionID, _, err = session.Create(sessionOptions, nil)
		return err
	}, backoff.NewExponentialBackOff(), func(err error, t time.Duration) {
		log.Println("Cannot create session, retrying in", t, ". Error:", err)
	})
	if err != nil {
		return fmt.Errorf("failed to create consul session: %v", err)
	}

	// set up a long-running goroutine for renewing the session
	c.sessionRenew = make(chan struct{})
	c.sessionID = sessionID
	log.Println("[coordinator] Coordinator ready")

	go func() {
		errCh <- session.RenewPeriodic("5s", sessionID, nil, c.sessionRenew)
	}()
	return nil
}

// non-blocking
func mustWatchServiceDefs(ctx context.Context, client etcd.KeysAPI, basepath *string, changed chan<- bool) {
	wOpts := &etcd.WatcherOptions{Recursive: true}
	watcher := client.Watcher(*basepath, wOpts)

	watchOperation := func() error {
		resp, err := watcher.Next(ctx)
		if err != nil {
			switch v := err.(type) {
			case etcd.Error:
				if v.Code == etcd.ErrorCodeEventIndexCleared {
					watcher = client.Watcher(*basepath, wOpts)

					log.WithFields(log.Fields{
						"basepath": *basepath,
						"code":     v.Code,
						"cause":    v.Cause,
						"index":    v.Index,
						"message":  v.Message,
					}).Warn("refreshed watcher")

					return nil
				}
			default:
				if err.Error() == "unexpected end of JSON input" {
					log.WithField("error", err).Warn("probably a connection timeout. are we in etcd 0.4.x?")
					return nil
				} else {
					return err
				}
			}
		}

		if resp.Action != "get" {
			changed <- true
		}

		return nil
	}

	notify := func(err error, dur time.Duration) {
		log.WithFields(log.Fields{
			"dur":          dur,
			"error":        err,
			"service_path": *basepath,
		}).Error("service definition watch failed. backing off.")
	}

	go func() {
		for {
			err := backoff.RetryNotify(watchOperation, backoff.NewExponentialBackOff(), notify)
			if err != nil {
				log.WithFields(log.Fields{
					"error":        err,
					"service_path": *basepath,
				}).Fatal("unable to recover communication with etcd, watch abandoned")
			}
		}
	}()
}

func main() {
	cfg, err := New()
	if err != nil {
		log.Fatalf("Failed to parse config: %s", err)
		return
	}

	runs := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "elasticsearch_backup_runs_total",
			Help: "Number of elasticsearch backup runs",
		},
		[]string{"status"},
	)
	runs = prometheus.MustRegisterOrGet(runs).(*prometheus.CounterVec)
	duration := prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Name: "elasticsearch_backup_duration",
			Help: "Duration of elasticsearch backup runs",
		},
		[]string{"operation"},
	)
	duration = prometheus.MustRegisterOrGet(duration).(*prometheus.SummaryVec)

	go listen()

	interval := time.Hour * time.Duration(cfg.Interval)
	for {
		t0 := time.Now()
		opFunc := func() error {
			return backupAndRemove(cfg)
		}
		logFunc := func(err error, wait time.Duration) {
			log.Warnf("Failed to connect to ES: %s. Retry in %s", err, wait)
		}
		bo := backoff.NewExponentialBackOff()
		bo.InitialInterval = time.Second
		bo.MaxInterval = 60 * time.Second
		bo.MaxElapsedTime = 15 * time.Minute
		log.Infof("Attempting Snapshot ...")
		err := backoff.RetryNotify(opFunc, bo, logFunc)
		if err != nil {
			runs.WithLabelValues("failed").Inc()
			log.Warnf("Failed to delete snapshots: %s", err)
			continue
		}
		runs.WithLabelValues("ok").Inc()
		d0 := float64(time.Since(t0)) / float64(time.Microsecond)
		duration.WithLabelValues("backup").Observe(d0)

		if interval < time.Second {
			break
		}
		log.Infof("Waiting %s until next run", interval.String())
		time.Sleep(interval)
	}
	os.Exit(0)
}

// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage) error {

	var dockerClient *docker.Client
	var err error

	if provider.TLS != nil {
		dockerClient, err = docker.NewTLSClient(provider.Endpoint,
			provider.TLS.Cert, provider.TLS.Key, provider.TLS.CA)
	} else {
		dockerClient, err = docker.NewClient(provider.Endpoint)
	}
	if err != nil {
		log.Errorf("Failed to create a client for docker, error: %s", err)
		return err
	}
	err = dockerClient.Ping()
	if err != nil {
		log.Errorf("Docker connection error %+v", err)
		return err
	}
	log.Debug("Docker connection established")
	if provider.Watch {
		dockerEvents := make(chan *docker.APIEvents)
		dockerClient.AddEventListener(dockerEvents)
		log.Debug("Docker listening")
		go func() {
			operation := func() error {
				for {
					event := <-dockerEvents
					if event == nil {
						return errors.New("Docker event nil")
						//							log.Fatalf("Docker connection error")
					}
					if event.Status == "start" || event.Status == "die" {
						log.Debugf("Docker event receveived %+v", event)
						configuration := provider.loadDockerConfig(dockerClient)
						if configuration != nil {
							configurationChan <- types.ConfigMessage{"docker", configuration}
						}
					}
				}
			}
			notify := func(err error, time time.Duration) {
				log.Errorf("Docker connection error %+v, retrying in %s", err, time)
			}
			err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
			if err != nil {
				log.Fatalf("Cannot connect to docker server %+v", err)
			}
		}()
	}

	configuration := provider.loadDockerConfig(dockerClient)
	configurationChan <- types.ConfigMessage{"docker", configuration}
	return nil
}

// Retry is the core library method for retrying http calls.
//
// httpCall should be a function that performs the http operation, and returns
// (resp *http.Response, tempError error, permError error). Errors that should
// cause retries should be returned as tempError. Permanent errors that should
// not result in retries should be returned as permError. Retries are performed
// using the exponential backoff algorithm from the github.com/cenkalti/backoff
// package. Retry automatically treats HTTP 5xx status codes as a temporary
// error, and any other non-2xx HTTP status codes as a permanent error. Thus
// httpCall function does not need to handle the HTTP status code of resp,
// since Retry will take care of it.
//
// Concurrent use of this library method is supported.
func (httpRetryClient *Client) Retry(httpCall func() (resp *http.Response, tempError error, permError error)) (*http.Response, int, error) {
	var tempError, permError error
	var response *http.Response
	attempts := 0
	doHttpCall := func() error {
		response, tempError, permError = httpCall()
		attempts += 1
		if tempError != nil {
			return tempError
		}
		if permError != nil {
			return nil
		}
		// only call this if there is a non 2xx response
		body := func(response *http.Response) string {
			// this is a no-op
			raw, err := httputil.DumpResponse(response, true)
			if err == nil {
				return string(raw)
			}
			return ""
		}
		// now check if http response code is such that we should retry [500, 600)...
		if respCode := response.StatusCode; respCode/100 == 5 {
			return BadHttpResponseCode{
				HttpResponseCode: respCode,
				Message:          "(Intermittent) HTTP response code " + strconv.Itoa(respCode) + "\n" + body(response),
			}
		}
		// now check http response code is ok [200, 300)...
		if respCode := response.StatusCode; respCode/100 != 2 {
			permError = BadHttpResponseCode{
				HttpResponseCode: respCode,
				Message:          "(Permanent) HTTP response code " + strconv.Itoa(respCode) + "\n" + body(response),
			}
			return nil
		}
		return nil
	}

	// Make HTTP API calls using an exponential backoff algorithm...
	b := backoff.ExponentialBackOff(*httpRetryClient.BackOffSettings)
	backoff.RetryNotify(doHttpCall, &b, func(err error, wait time.Duration) {
		log.Printf("Error: %s", err)
	})

	switch {
	case permError != nil:
		return response, attempts, permError
	case tempError != nil:
		return response, attempts, tempError
	default:
		return response, attempts, nil
	}
}

func withRetries(f func() error) error {
	backoffConfig := backoff.NewExponentialBackOff()
	backoffConfig.InitialInterval = time.Second
	backoffConfig.MaxInterval = 10 * time.Second
	backoffConfig.MaxElapsedTime = 60 * time.Second

	notifyFunc := func(err error, dur time.Duration) {
		log.Printf("waiting %v, failed to get move from player: %s", dur, err)
	}
	err := backoff.RetryNotify(f, backoffConfig, notifyFunc)
	return err
}

func (c *Client) CleanTorrents() error {
	logger.Infof("Running torrent cleaner")
	torrents, er := c.GetTorrents()
	if er != nil {
		return er
	}

	torrents.SortByID(false)
	logger.Infof("Found %d torrents to process", len(torrents))
	for _, t := range torrents {
		logger.Debugf("[Torrent %d: %q] Checking status", t.ID, t.Name)
		id := util.Hashf(md5.New(), t.ID, t.Name)
		status := &torrentStatus{Torrent: t, id: id, failures: 0}
		status.setFailures()

		if st, ok := seen[id]; ok {
			status.failures = status.failures + st.failures
			if !updated(st.Torrent, status.Torrent) {
				status.failures++
			}
		}

		seen[id] = status
		logger.Debugf("[Torrent %d: %q] Failures: %d", t.ID, t.Name, status.failures)
	}

	b := backoff.NewExponentialBackOff()
	b.MaxElapsedTime = 15 * time.Second
	remove := make([]*torrentStatus, 0, 1)
	for _, t := range seen {
		if t.failed() {
			b.Reset()
			logger.Infof("[Torrent %d: %q] Removing", t.ID, t.Name)
			er := backoff.RetryNotify(delTorrent(c, t.Torrent), b, func(e error, w time.Duration) {
				logger.Errorf("[Torrent %d: %q] Failed to remove (retry in %v): %v", t.ID, t.Name, w, e)
			})
			if er == nil {
				remove = append(remove, t)
			} else {
				logger.Errorf("[Torrent %d: %q] Failed to remove, will retry next cycle", t.ID, t.Name)
			}
		}
	}

	for i := range remove {
		delete(seen, remove[i].id)
	}
	return nil
}

// RoundTrip implements the RoundTripper interface.
func (t *BackOffTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	// Initialize the exponential backoff client.
	backOffClient := &backoff.ExponentialBackOff{
		InitialInterval:     t.backOffConfig.initialInterval,
		RandomizationFactor: t.backOffConfig.randomizationFactor,
		Multiplier:          t.backOffConfig.backOffMultiplier,
		MaxInterval:         t.backOffConfig.maxInterval,
		MaxElapsedTime:      t.backOffConfig.maxElapsedTime,
		Clock:               backoff.SystemClock,
	}
	// Make a copy of the request's Body so that we can reuse it if the request
	// needs to be backed off and retried.
	bodyBuf := bytes.Buffer{}
	if req.Body != nil {
		if _, err := bodyBuf.ReadFrom(req.Body); err != nil {
			return nil, fmt.Errorf("Failed to read request body: %v", err)
		}
	}

	var resp *http.Response
	var err error
	roundTripOp := func() error {
		if req.Body != nil {
			req.Body = ioutil.NopCloser(bytes.NewBufferString(bodyBuf.String()))
		}
		resp, err = t.Transport.RoundTrip(req)
		if err != nil {
			return fmt.Errorf("Error while making the round trip: %s", err)
		}
		if resp != nil {
			if resp.StatusCode >= 500 && resp.StatusCode <= 599 {
				return fmt.Errorf("Got server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL)
			} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
				// Stop backing off if there are non server errors.
				backOffClient.MaxElapsedTime = backoff.Stop
				return fmt.Errorf("Got non server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL)
			}
		}
		return nil
	}
	notifyFunc := func(err error, wait time.Duration) {
		glog.Warningf("Got error: %s. Retrying HTTP request after sleeping for %s", err, wait)
	}

	if err := backoff.RetryNotify(roundTripOp, backOffClient, notifyFunc); err != nil {
		return nil, fmt.Errorf("HTTP request failed inspite of exponential backoff: %s", err)
	}
	return resp, nil
}

func restartProcesses(t, v *process.Process, c *config.Config, ctx context.Context) error {
	var (
		notify = func(e error, t time.Duration) {
			logger.Errorf("Failed to restart processes (retry in %v): %v", t, e)
		}
		operation = func() error {
			t.Stop()
			v.Stop()
			return startProcesses(t, v, c, ctx)
		}
		b = backoff.NewExponentialBackOff()
	)
	b.MaxElapsedTime = 30 * time.Minute
	b.MaxInterval = 10 * time.Second
	return backoff.RetryNotify(operation, b, notify)
}

func getIP(dev string, timeout time.Duration, c context.Context) (string, error) {
	var address string
	notify := func(e error, w time.Duration) {
		logger.Errorf("Failed to get IP for %q (retry in %v): %v", dev, w, e)
	}
	fn := func() (er error) {
		select {
		default:
			address, er = vpn.FindIP(dev)
			return
		case <-c.Done():
			return
		}
	}

	b := backoff.NewExponentialBackOff()
	b.MaxElapsedTime = timeout
	return address, backoff.RetryNotify(fn, b, notify)
}

// RegionLeader block indefinitely until this invocation has been elected the "leader" within the local operating region.
// It will then return a channel that will eventually be closed when leadership is rescinded.
func RegionLeader(id string) Leader {
	path := fmt.Sprintf(regionLeaderPath, id)
	prefix := path + "/lock-"
	var lockNode string

	for {
		// create our lock node -- retry until this is done, use exponential backoff
		// to add some delay between attempts
		b := backoff.NewExponentialBackOff()
		b.InitialInterval = backoffInitialInterval
		b.MaxInterval = backoffMaxInterval
		b.MaxElapsedTime = 0 // Never stop retrying

		backoff.RetryNotify(func() (err error) {
			log.Infof("[Sync:RegionLeader] Attepting to create ephemeral lock node for leadership election")
			lockNode, err = zookeeper.CreateProtectedEphemeralSequential(prefix, []byte{}, gozk.WorldACL(gozk.PermAll))

			return
		}, b, func(err error, d time.Duration) {
			if err == gozk.ErrNoNode {
				createParents(path)
			} else if err != nil {
				log.Warnf("[Sync:RegionLeader] ZooKeeper error creating ephemeral lock node for leadership election: %s. Waiting %s", err, d)
			}
		})

		err := waitForWinner(path, lockNode)
		if err != nil {
			// try to cleanup - then go again
			zookeeper.Delete(lockNode, -1)
			time.Sleep(time.Second)
			continue
		}

		// we are the leader
		break
	}

	log.Infof("[Sync:RegionLeader] Elected leader of '%v'", id)
	inst.Counter(1.0, "sync.regionleader.elected")

	return newRegionLeader(lockNode)
}

// DoCommand executes the specific rpc command wrapped in rpcFunc.
func (c *Connection) DoCommand(ctx context.Context, rpcFunc func(keybase1.GenericClient) error) error {
	for {
		// we may or may not be in the process of reconnecting.
		// if so we'll block here unless canceled by the caller.
		connErr := c.waitForConnection(ctx)
		if connErr != nil {
			return connErr
		}

		var rpcErr error

		// retry throttle errors w/backoff
		throttleErr := backoff.RetryNotify(func() error {
			rawClient := func() keybase1.GenericClient {
				c.mutex.Lock()
				defer c.mutex.Unlock()
				return c.client
			}()
			// try the rpc call. this can also be canceled
			// by the caller, and will retry connectivity
			// errors w/backoff.
			throttleErr := runUnlessCanceled(ctx, func() error {
				return rpcFunc(rawClient)
			})
			if c.handler.ShouldThrottle(throttleErr) {
				return throttleErr
			}
			rpcErr = throttleErr
			return nil
		}, backoff.NewExponentialBackOff(), c.handler.OnDoCommandError)

		// RetryNotify gave up.
		if throttleErr != nil {
			return throttleErr
		}

		// check to see if we need to retry it.
		if !c.checkForRetry(rpcErr) {
			return rpcErr
		}
	}
}