func (pc *PathCache) Start() (notifyChan chan PathCacheNotification, err error) {
	pc.notifyChan = make(chan PathCacheNotification, 10)
	pc.stoppedChan = make(chan bool, 1)
	pc.startup = sync.WaitGroup{}

	go pc.mux()

	err = pc.watch()

	if err != nil {
		// We expect this to occasionally happen due to timing
		if err != zk.ErrNoNode {
			log.Println(log.ERROR, err)
		}

		pc.Stop()
		return
	}

	err = pc.watchChildren()

	if err != nil {
		// We expect this to occasionally happen due to timing
		if err != zk.ErrNoNode {
			log.Println(log.ERROR, err)
		}
		pc.Stop()
	}

	pc.startup.Wait()

	return pc.notifyChan, err
}

func (s *SkynetDaemon) StopAllSubServices(requestInfo *skynet.RequestInfo, in daemon.StopAllSubServicesRequest, out *daemon.StopAllSubServicesResponse) (err error) {
	var uuids []string
	s.serviceLock.Lock()
	for uuid := range s.Services {
		uuids = append(uuids, uuid)
	}
	s.serviceLock.Unlock()

	out.Stops = make([]daemon.StopSubServiceResponse, len(uuids))

	for i, uuid := range uuids {
		log.Println(log.TRACE, "Stopping "+uuid)
		err = s.StopSubService(requestInfo, daemon.StopSubServiceRequest{UUID: uuid}, &out.Stops[i])
		if err != nil {
			log.Println(log.ERROR, "Failed to stop subservice "+uuid, err)
			return
		}
		if out.Stops[i].Ok {
			out.Count++
		}
	}

	s.saveState()

	return
}

// TODO: This should be moved out so that it's run asynchronously
// it should also use a buffered channel so that if a save is already queued it only saves once
func (s *SkynetDaemon) writeStateFile() (err error) {
	err = s.stateFile.Truncate(0)

	if err != nil {
		return
	}

	_, err = s.stateFile.Seek(0, 0)

	if err != nil {
		return
	}

	var b []byte
	b, err = json.MarshalIndent(s.Services, "", "\t")

	if err != nil {
		log.Println(log.ERROR, "Failed to marshall daemon state")
		return
	}

	_, err = s.stateFile.Write(b)

	if err != nil {
		log.Println(log.ERROR, "Failed to save daemon state")
	}

	return
}

func (c *ServiceClient) send(retry, giveup time.Duration, ri *skynet.RequestInfo, fn string, in interface{}, out interface{}) (err error) {
	if ri == nil {
		ri = c.NewRequestInfo()
	}

	attempts := make(chan sendAttempt)

	var retryTicker <-chan time.Time
	retryChan := make(chan bool)
	if retry > 0 {
		retryTicker = time.Tick(retry)
	}

	var timeoutTimer <-chan time.Time
	if giveup > 0 {
		timeoutTimer = time.NewTimer(giveup).C
	}

	attemptCount := 1
	go c.attemptSend(retry, attempts, ri, fn, in, out)

	for {
		select {
		case <-retryTicker:
			retryChan <- true

		case <-retryChan:
			attemptCount++
			ri.RetryCount++
			log.Println(log.TRACE, fmt.Sprintf("Sending Attempt# %d with RequestInfo %+v", attemptCount, ri))
			go c.attemptSend(retry, attempts, ri, fn, in, out)

		case <-timeoutTimer:
			err = RequestTimeout
			return

		case attempt := <-attempts:
			if attempt.err != nil {
				log.Println(log.ERROR, "Attempt Error: ", attempt.err)

				// If there is no retry timer we need to exit as retries were disabled
				if retryTicker == nil {
					return err
				} else {
					// Don't wait for next retry tick retry now
					retryChan <- true
				}

				continue
			}

			// copy into the caller's value
			v := reflect.Indirect(reflect.ValueOf(out))
			v.Set(reflect.Indirect(reflect.ValueOf(attempt.result)))

			return
		}
	}
}

// this function is the goroutine that owns this service - all thread-sensitive data needs to
// be manipulated only through here.
func (s *Service) mux() {
loop:
	for {
		select {
		case conn := <-s.connectionChan:
			clientID := config.NewUUID()

			s.clientMutex.Lock()
			s.ClientInfo[clientID] = ClientInfo{
				Address: conn.RemoteAddr(),
			}
			s.clientMutex.Unlock()

			// send the server handshake
			sh := skynet.ServiceHandshake{
				Registered: s.Registered,
				ClientID:   clientID,
				Name:       s.Name,
			}

			encoder := bsonrpc.NewEncoder(conn)
			err := encoder.Encode(sh)
			if err != nil {
				log.Println(log.ERROR, "Failed to encode server handshake", err.Error())
				continue
			}
			if !s.Registered {
				log.Println(log.ERROR, "Connection attempted while unregistered. Closing connection")
				conn.Close()
				continue
			}

			// read the client handshake
			var ch skynet.ClientHandshake
			decoder := bsonrpc.NewDecoder(conn)
			err = decoder.Decode(&ch)
			if err != nil {
				log.Println(log.ERROR, "Error calling bsonrpc.NewDecoder: "+err.Error())
				continue
			}

			// here do stuff with the client handshake
			go func() {
				s.RPCServ.ServeCodec(bsonrpc.NewServerCodec(conn))
			}()
		case register := <-s.registeredChan:
			if register {
				s.register()
			} else {
				s.unregister()
			}
		case <-s.shutdownChan:
			s.shutdown()
		case _ = <-s.doneChan:
			break loop
		}
	}
}

func (sc *scodec) ReadRequestHeader(rq *rpc.Request) (err error) {
	log.Println(log.TRACE, "RPC Server Entered: ReadRequestHeader")
	defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestHeader")

	err = sc.dec.Decode(rq)
	if err != nil && err != io.EOF {
		log.Println(log.ERROR, "RPC Server Error decoding request header: ", err)
	}
	return
}

func (sc *scodec) ReadRequestBody(v interface{}) (err error) {
	log.Println(log.TRACE, "RPC Server Entered: ReadRequestBody")
	defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestBody")

	err = sc.dec.Decode(v)
	if err != nil {
		log.Println(log.ERROR, "RPC Server Error decoding request body: ", err)
	}
	return
}

func (sc *scodec) Close() (err error) {
	log.Println(log.TRACE, "RPC Server Entered: Close")
	defer log.Println(log.TRACE, "RPC Server Leaving: Close")

	err = sc.conn.Close()
	if err != nil {
		log.Println(log.ERROR, "RPC Server Error closing connection: ", err)
		return
	}
	return
}

func (cc *ccodec) ReadResponseHeader(res *rpc.Response) (err error) {
	log.Println(log.TRACE, "RPC Server Entered: ReadResponseHeader")
	defer log.Println(log.TRACE, "RPC Server Leaving: ReadResponseHeader")

	err = cc.dec.Decode(res)

	if err != nil {
		log.Println(log.ERROR, "RPC Client Error decoding response header: ", err)
	}
	return
}

func (sd *SkynetDaemon) Started(s *service.Service) {
	err := sd.cleanupHost(s.ServiceInfo.UUID)
	if err != nil {
		log.Println(log.ERROR, "Error cleaning up host", err)
	}

	err = sd.restoreState()

	if err != nil {
		log.Println(log.ERROR, "Error restoring state", err)
	}
}

func getGiveupTimeout(service, version string) time.Duration {
	if d, err := config.String(service, version, "client.timeout.total"); err == nil {
		if timeout, err := time.ParseDuration(d); err == nil {
			log.Println(log.TRACE, fmt.Sprintf("Using custom giveup duration %q for %q %q", timeout.String(), service, version))
			return timeout
		}

		log.Println(log.ERROR, "Failed to parse client.timeout.total", err)
	}

	return config.DefaultRetryDuration
}

/*
Conn.SendTimeout() Acts like Send but takes a timeout
*/
func (c *Conn) SendTimeout(ri *skynet.RequestInfo, fn string, in interface{}, out interface{}, timeout time.Duration) (err error) {
	if c.IsClosed() {
		return ConnectionClosed
	}

	sin := skynet.ServiceRPCInWrite{
		RequestInfo: ri,
		Method:      fn,
		ClientID:    c.clientID,
	}

	var b []byte
	b, err = bson.Marshal(in)
	if err != nil {
		return serviceError{fmt.Sprintf("Error calling bson.Marshal: %v", err)}
	}

	sin.In = bson.Binary{
		0x00,
		b,
	}

	sout := skynet.ServiceRPCOutRead{}

	// Set timeout for this request, then set it back to idle timeout
	c.setDeadline(timeout)
	defer c.setDeadline(c.idleTimeout)

	log.Println(log.TRACE, fmt.Sprintf("Sending Method call %s with ClientID %s to: %s", sin.Method, sin.ClientID, c.addr))
	err = c.rpcClient.Call(c.serviceName+".Forward", sin, &sout)
	if err != nil {
		c.Close()
		err = serviceError{err.Error()}

		return
	}

	if sout.ErrString != "" {
		err = serviceError{sout.ErrString}
		return
	}

	err = bson.Unmarshal(sout.Out, out)
	if err != nil {
		log.Println(log.ERROR, "Error unmarshalling nested document")
		err = serviceError{err.Error()}
	}

	return
}

func init() {
	flagset := flag.NewFlagSet("config", flag.ContinueOnError)
	flagset.StringVar(&configFile, "config", "", "Config File")
	flagset.StringVar(&uuid, "uuid", "", "uuid")

	args, _ := SplitFlagsetFromArgs(flagset, os.Args[1:])
	flagset.Parse(args)

	// Ensure we have a UUID
	if uuid == "" {
		uuid = NewUUID()
	}

	if configFile == "" {
		for _, f := range defaultConfigFiles {
			if _, err := os.Stat(f); err == nil {
				configFile = f
				break
			}
		}
	}

	if configFile == "" {
		log.Println(log.ERROR, "Failed to find config file")
		conf = config.NewDefault()
		return
	}

	if _, err := os.Stat(configFile); os.IsNotExist(err) {
		log.Println(log.ERROR, "Config file does not exist", err)
		conf = config.NewDefault()
		return
	}

	var err error
	if conf, err = config.ReadDefault(configFile); err != nil {
		conf = config.NewDefault()
		log.Fatal(err)
	}

	// Set default log level from config, this can be overriden at the service level when the service is created
	if l, err := conf.RawStringDefault("log.level"); err == nil {
		log.SetLogLevel(log.LevelFromString(l))
	}

	// Set GOMAXPROCS
	if i, err := conf.Int("DEFAULT", "runtime.gomaxprocs"); err == nil {
		runtime.GOMAXPROCS(i)
	}
}

// Wait for existing requests to complete and shutdown service
func (s *Service) shutdown() {
	if s.shuttingDown {
		return
	}

	s.shuttingDown = true

	s.doneGroup.Add(1)
	s.rpcListener.Close()

	s.doneChan <- true

	s.activeRequests.Wait()

	err := skynet.GetServiceManager().Remove(*s.ServiceInfo)
	if err != nil {
		log.Println(log.ERROR, "Failed to remove service: "+err.Error())
	}

	skynet.GetServiceManager().Shutdown()

	s.Delegate.Stopped(s) // Call user defined callback

	s.doneGroup.Done()
}

func (sm *ZookeeperServiceManager) Remove(s skynet.ServiceInfo) (err error) {
	log.Println(log.TRACE, "Removing service", s.UUID)

	ops := zk.MultiOps{
		Delete: []zk.DeleteRequest{
			deleteRequest(path.Join("/regions", s.Region, s.UUID), -1),
			deleteRequest(path.Join("/services", s.Name, s.Version, s.UUID), -1),
			deleteRequest(path.Join("/services", s.Name, s.UUID), -1),
			deleteRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), -1),

			deleteRequest(path.Join("/instances", s.UUID, "registered"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "name"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "version"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "region"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "addr"), -1),

			deleteRequest(path.Join("/instances", s.UUID), -1),
		},
	}

	err = sm.conn.Multi(ops)

	if err == nil {
		delete(sm.managedInstances, s.UUID)
	}

	// Attempt to remove parent paths for service if they are empty
	sm.removePathIfEmpty(path.Join("/hosts", s.ServiceAddr.IPAddress))
	sm.removePathIfEmpty(path.Join("/regions", s.Region))
	sm.removePathIfEmpty(path.Join("/services", s.Name, s.Version))
	sm.removePathIfEmpty(path.Join("/services", s.Name))

	return
}

func (sm *ZookeeperServiceManager) Add(s skynet.ServiceInfo) (err error) {
	log.Println(log.TRACE, "Adding service to cluster", s.UUID)

	err = sm.createPathsForService(s)

	if err != nil {
		return err
	}

	ops := zk.MultiOps{
		Create: []zk.CreateRequest{
			createRequest(path.Join("/regions", s.Region, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/services", s.Name, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/services", s.Name, s.Version, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID), []byte{}, zk.PermAll, 0),

			createRequest(path.Join("/instances", s.UUID, "registered"), []byte(strconv.FormatBool(s.Registered)), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "name"), []byte(s.Name), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "version"), []byte(s.Version), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "region"), []byte(s.Region), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "addr"), []byte(s.ServiceAddr.String()), zk.PermAll, zk.FlagEphemeral),
		},
	}

	err = sm.conn.Multi(ops)

	sm.managedInstances[s.UUID] = s

	return
}

func (s *Service) listen(addr skynet.BindAddr, bindWait *sync.WaitGroup) {
	var err error
	s.rpcListener, err = addr.Listen()
	if err != nil {
		panic(err)
	}

	log.Printf(log.INFO, "%+v\n", ServiceListening{
		Addr:        &addr,
		ServiceInfo: s.ServiceInfo,
	})

	// We may have changed port due to conflict, ensure config has the correct port now
	a, _ := skynet.BindAddrFromString(addr.String())
	s.ServiceAddr.IPAddress = a.IPAddress
	s.ServiceAddr.Port = a.Port

	bindWait.Done()

	for {
		conn, err := s.rpcListener.AcceptTCP()

		if s.shuttingDown {
			break
		}

		if err != nil && !s.shuttingDown {
			log.Println(log.ERROR, "AcceptTCP failed", err)
			continue
		}
		s.connectionChan <- conn
	}
}

func (sc *scodec) WriteResponse(rs *rpc.Response, v interface{}) (err error) {
	log.Println(log.TRACE, "RPC Server Entered: WriteResponse")
	defer log.Println(log.TRACE, "RPC Server Leaving: WriteResponse")

	err = sc.enc.Encode(rs)
	if err != nil {
		log.Println(log.ERROR, "RPC Server Error encoding rpc response: ", err)
		return
	}
	err = sc.enc.Encode(v)
	if err != nil {
		log.Println(log.ERROR, "RPC Server Error encoding response value: ", err)
		return
	}
	return sc.encBuf.Flush()
}

func (sm *ZookeeperServiceManager) Unregister(uuid string) (err error) {
	log.Println(log.TRACE, "Unregister service", uuid)

	_, err = sm.conn.Set(path.Join("/instances", uuid, "registered"), []byte("false"), -1)

	return
}

func (sm *ZookeeperServiceManager) mux() {
	for {
		select {
		case e := <-sm.session:
			switch e.Type {
			case zk.EventNodeDeleted, zk.EventNodeChildrenChanged, zk.EventNodeDataChanged:
			case zk.EventSession:
			// TODO: EventNotWatching
			// TODO: StateDisconnected
			default:
				log.Println(log.TRACE, "Zookeeper Event Received: ", e)
			}
		case <-sm.done:
			// Remove instances that were added by this instance
			for _, s := range sm.managedInstances {
				sm.Remove(s)
			}

			sm.cache.Stop()

			sm.conn.Close()
			return
		}
	}
}