def events_vimeo_analytics():
   #
   # we should be aware that engineering events get submitted through various
   # means (incl scripts), so if category is not set, but expected FX is, we
   # count those as engineering

    backend = Backend()
    events = []
    query = {
        "query_string": {
            "fields": ["category"],
            "query": "*"
        }
    }

    for event in backend.get_events_raw(query):
        desc = event['desc'].replace("\n", '  ').replace("\r", ' ').strip()
        tags = '-'.join(event['tags'])
        expected_effect = event.get('expected_effect', '')
        if type(expected_effect) is list:
            expected_effect = '-'.join(expected_effect)
        known_effect = event.get('known_effect', '')
        event = [event['id'], event['date'], desc, tags, event['category'], expected_effect, known_effect]
        events.append(event)
    return events

class TranslateBot(JabberBot):
   """My Translator Bot"""
   
   def __init__(self, username, password, res=None, debug=False):
      self.backend = Backend()
      JabberBot.__init__(self, username, password, res, debug)
   
   @botcmd
   def bot_time(self, mess, args):
      """Displays current server time"""
      return str(datetime.datetime.now())
   
   @botcmd
   def version(self, mess, args):
      """print current version of module"""
      return self.backend.version(mess, args)
   	  
   @botcmd(hidden=True)
   def reload(self, mess, args):
      """DEBUG: reload module"""
      import backend
      reload(backend)
      self.backend = backend.Backend()
      return "module reloaded."

   @botcmd
   def translate(self, mess, args):      
      """translate: TODO add help"""
      return self.backend.translate(mess, args)
   
   @botcmd(hidden=True, default=True)
   def trans(self, mess, args):
      return self.backend.translate(mess, args) 

class Store(LineReceiver):

    def __init__(self, db):
        # each TCP request handler gets its own sqlite cursor
        # this should be sufficient for proper concurrent behavior,
        # but if anyone can confirm this, that would be appreciated
        self.backend = Backend(db, exists=True)
        # support both \n and (default) \r\n delimiters
        # http://stackoverflow.com/questions/5087559/twisted-server-nc-client
        self.delimiter = "\n"

    def lineReceived(self, line):
        # line: <timestamp> <some> <tags=here> -- description text of the event goes here
        event = line.rstrip("\r").split(" -- ", 1)
        event[0] = event[0].split(' ', 1)
        timestamp = int(event[0][0])
        tags = event[0][1].split(' ')
        desc = event[1]
        try:
            event = Event(timestamp=timestamp, desc=desc, tags=tags)
            print "line:", line
            self.backend.add_event(event)
        except sqlite3.OperationalError, e:
            sys.stderr.write("sqlite error, aborting. : %s" % e)
            sys.exit(2)
        except Exception, e:
            sys.stderr.write("bad line: %s  --> error: %s\n" % (line, e))

    def __init__(self, *fargs, **args):
        Backend.__init__(self, *fargs, **args)

        if "log_path" in args:
            if "debug" in args:
                self.debug = args["debug"]
            else:
                self.debug = True
        else:
            self.debug = False

        if self.debug:
            self.setup_logfile("dbpedia", args["log_path"])

        if "backend" in args:
            self.set_backend(args["backend"])
        else:
            self.set_backend(self.default_backend)
        
        if len(fargs) > 1:
            if type(fargs[0]) == str:
                self.request = fargs
            else:
                print('fixme', fargs)
                sys.exit()
        else:
            self.request = fargs
        if type(self.request)  == tuple:
            self.request = self.request[0]

def sch(request, pi_id):
    print("In sched")
    sch_id = request.POST['sch_id']
    print repr(request.POST)
    action = request.POST['action']
    schedule = Schedule.objects.get(pk=sch_id)
    print "Before action '%s'" % action
    be = Backend()
    print "Backend stat size: %s" % len(be.schedDict)

    if action == "stop":
        status = Const.STATUS_STOPPED
        # cancel currently running schedule and start new
        schedule_current = Schedule.objects.filter(Q(status=Const.STATUS_RUNNING) | Q(status=Const.STATUS_PLANNED))
        if len(schedule_current) > 0:
            print "Got running schedule '%s'" % schedule_current[0]
            schedule_current[0].status = Const.STATUS_STOPPED
            schedule_current[0].save()
            print "Forcing stop scheduler"
            be.stop_schedule(schedule_current[0])
    elif action == "start":
        status = Const.STATUS_RUNNING
        be.exec_schedule(schedule)

    print "Status switched to: %s" % status % ", action: %s" % action
    schedule.status = status
    schedule.save()
    print("Saved")

    message = "Schedule(" + sch_id + ") is %s" % ( "started" if status == Const.STATUS_RUNNING else "stopped")
    print("Message:" + message)
    return HttpResponse(message)

def main():
	import argparse

	parser = argparse.ArgumentParser(description='set password for AppleID. CAUTION: Use AppleIDs with payment credentials at you own risk!!! EVERY purchase will be done if possible!!! The password will be stored UNENCRYPTED!!!')
	parser.add_argument('-b','--backend', required=True, help='the backend url')
	parser.add_argument('-a','--appleId', required=True, help='the AppleId')
	parser.add_argument('-p','--password', required=True, help='the password')

	args = parser.parse_args()
	logger.debug(args)

	backend = Backend(args.backend)
	accounts = backend.get_accounts()
	
	passwordUpdated = False
	for accId, acc in accounts.items():
		if 'appleId' in acc and acc['appleId'] == args.appleId:
			logger.debug(str(acc))
			acc['password'] = args.password
			passwordUpdated = backend.post_account(acc)
			break
	
	if passwordUpdated:
		print "password updated for AppleId '%s'" % args.appleId
	else:
		print "unable to update password for AppleId '%s'" % args.appleId

def autoupdate():
    """
    Autoupdate data layer about new models in engines.
    """
    scoped_session(Backend.instance().get_sessionmaker)
    Backend.instance().get_base().metadata.create_all(
        Backend.instance().get_engine())

class StoreFactory(Factory):

    def __init__(self, db):
        self.db = db
        self.backend = Backend(db)
        try:
            self.backend.assure_db()
        except sqlite3.OperationalError, e:
            sys.stderr.write("sqlite error, aborting. : %s" % e)
            sys.exit(2)

 def __init__(self, engine_config):
   Backend.__init__(self, engine_config)
   try:
     self.con = MySQLdb.connect(host=dbhost,
                                user=dbuser,
                                passwd=dbpass,
                                db=dbname,
                                charset='utf8')
     self.logger.info("Connected to MySQL db %s:%s." % (dbhost, dbname))
     self.cur = self.con.cursor()
   except Exception as e:
     raise BackendError("Error connecting to MySQL db %s:%s: %s" % (dbhost, dbname, e))

def stop_schedule(request, pi_id):
    be = Backend()
    messsage = {"Error"}
    try:
        sch_id = request.POST['sch_id']
        # # schedule = change_schedule_status(sch_id, STATUS_STOPPED)
        schedule = get_object_or_404(Schedule, id=sch_id)
        be.stop_schedule(schedule)
        messsage = {Const.STATUS_STOPPED}
    except Schedule.DoesNotExist:
        raise Http404
    return HttpResponse(messsage)

def add_product_get(**kwargs):
    backend = Backend()
    return page(config, backend, state,
                body=template(
                    'tpl/events_add',
                    tags=backend.get_tags(),
                    extra_attributes=extra_attributes['product'],
                    event_type='product',
                    helptext=helptext['product'],
                    recommended_tags=[],
                    handler='vimeo_product',
                    **kwargs),
                page='add_product', **kwargs)

 def func_authenticate(self, _chan, *args, **kw):
     backend = Backend()
     if not backend.authenticate(username=kw['username'],
                                     auth_tokens=kw,
                                     ip_addr=kw['ip_addr']):
         return False
     else:
         if not self.namespaces.has_key(_chan):
             self.namespaces[_chan] = {}
         if not self.backend.has_key(_chan):
             self.backend[_chan] = backend
         self.namespaces[_chan]['client'] = backend.get_client_tags()
         return True

def add_analytics_get(**kwargs):
    backend = Backend()
    return page(config, backend, state,
                body=template(
                    'tpl/events_add',
                    tags=backend.get_tags(),
                    extra_attributes=extra_attributes['analytics'],
                    event_type='analytics',
                    helptext=helptext['analytics'],
                    recommended_tags=[],
                    handler='vimeo_analytics',
                    timestamp_feeder=True,
                    **kwargs),
                page='add_analytics', **kwargs)

def backup_az(az_domain, backup_az_domain, ceph_host, backup_ceph_host):
    # get ceph conf and keyring
    LOG.info("connect to ceph: host=%s" % ceph_host)
    ceph = Ceph(ceph_host=ceph_host, ceph_user=CEPH_USER, ceph_key_file=CEPH_KEYPAIR)

    LOG.info("get %s from %s" % (CEPH_CONF, ceph_host))
    ceph.get_file(LOCAL_CEPH_PATH+"/"+CEPH_CONF, REMOTE_CEPH_PATH+"/"+CEPH_CONF)

    LOG.info("get %s from %s" % (CEPH_KEYRING, ceph_host))
    ceph.get_file(LOCAL_CEPH_PATH+"/"+CEPH_KEYRING, REMOTE_CEPH_PATH+"/"+CEPH_KEYRING)
    ceph.close()

    # get backup ceph conf and keyring
    LOG.info("connect to backup_ceph: host=%s" % backup_ceph_host)
    backup_ceph = Ceph(ceph_host=backup_ceph_host, ceph_user=CEPH_USER, ceph_key_file=CEPH_KEYPAIR)
    
    LOG.info("get %s from %s" % (CEPH_BACKUP_CONF, backup_ceph_host))
    backup_ceph.get_file(LOCAL_CEPH_PATH+"/"+CEPH_BACKUP_CONF, REMOTE_CEPH_PATH+"/"+CEPH_BACKUP_CONF)
    
    LOG.info("get %s from %s" % (CEPH_BACKUP_KEYRING, backup_ceph_host))
    backup_ceph.get_file(LOCAL_CEPH_PATH+"/"+CEPH_BACKUP_KEYRING, REMOTE_CEPH_PATH+"/"+CEPH_BACKUP_KEYRING)
    backup_ceph.close()

    backend = Backend()
    # update volume_backend_name
    volume_backend_name = CEPH_VOLUME_PREFIX+":"+az_domain+":"+backup_az_domain
    LOG.info("ceph storage backend update: volume_backend_name = %s" % volume_backend_name)
    backend.update_ceph_param("volume_backend_name", volume_backend_name)

    # update iscsi_server_ip
    LOG.info("ceph storage backend update:iscsi_server_ip=%s" % ceph_host)
    backend.update_ceph_param("iscsi_server_ip", ceph_host)
    # backend.commit()
    '''
    update_params = {}
    volume_backend_name = CEPH_VOLUME_PREFIX+":"+az_domain+":"+backup_az_domain
    update_params["volume_backend_name"] = volume_backend_name
    update_params["iscsi_server_ip"] = ceph_host
    backend.update_ceph_params(update_params)
    '''
    # set volume_type key
    # volume_type=VOLUME_TYPE_PREFIX+"@"+az_domain
    shell_file = CURRENT_PATH+"/script/volume_backend_name.sh"
    # os.system("/bin/bash " + shell_file + " " + volume_type + " " + volume_backend_name)
    os.system("/bin/bash " + shell_file + " " + az_domain + " " + backup_az_domain)

    # restart Service
    restart_component("cinder", "cinder-volume")
    restart_component("cinder", "cinder-backup")

    def backend(self):
        '''Return D-BUS backend client interface.

        This gets initialized lazily.

        Set self.search_only to True to suppress a full system hardware
        detection, especially if you use only search_driver() to
        find a remote driver for a selected, already detected device.
        '''
        if self._dbus_iface is None:
            try:
                if self.argv_options.no_dbus:
                    self._dbus_iface = Backend()
                else:
                    self._dbus_iface = Backend.create_dbus_client()
            except Exception as e:
                if hasattr(e, '_dbus_error_name') and e._dbus_error_name in (
                    'org.freedesktop.DBus.Error.FileNotFound',
                    'org.freedesktop.DBus.Error.NoServer'):
                    if self.have_ui:
                        self.error_message(self._('Cannot connect to D-BUS,'+\
                            ' please use the --no-dbus option as root to'+\
                            ' use jockey without it.'),
                            str(e))
                    else:
                        self.error_msg(str(e))
                    sys.exit(1)
                else:
                    raise
            self._check_repositories()
            self._call_progress_dialog(
                self._('Searching for available drivers...'),
                self.search_only and self._dbus_iface.db_init or self._dbus_iface.detect, 
                timeout=600)
        else:
            # handle backend timeouts
            try:
                self._dbus_iface.handler_info(' ')
            except Exception as e:
                if hasattr(e, '_dbus_error_name') and e._dbus_error_name == \
                    'org.freedesktop.DBus.Error.ServiceUnknown':
                    self._dbus_iface = Backend.create_dbus_client()
                    self._check_repositories()
                    self._call_progress_dialog(
                        self._('Searching for available drivers...'),
                        self.search_only and self._dbus_iface.db_init or self._dbus_iface.detect, 
                        timeout=600)

        return self._dbus_iface

def start_schedule(request, pi_id):
    be = Backend()
    messsage = {"Error"}
    try:
        sch_id = request.POST['sch_id']
        # schedule = change_schedule_status(sch_id, STATUS_PLANNED)
        schedule = get_object_or_404(Schedule, id=sch_id)
        # Defense against idiot - do not allow to plan schedule multiple times if now, so process only active schedules
        if schedule.status is not Const.STATUS_RUNNING or schedule.status is not Const.STATUS_PLANNED:
            be.exec_schedule(schedule)
            messsage = {Const.STATUS_PLANNED}
        else:
            messsage = {"Nothing"}
    except Schedule.DoesNotExist:
        raise Http404
    return HttpResponse(messsage)

 def init(self):
     """
     Override parent to do explicit setting of the date format. (Oracle
     specific)
     """
     # Set date format
     self.setDateFormat("YYYY-MM-DD HH24:MI:SS")
     return Backend.init(self)

def do_action(request, pi_id):
    try:
        act_id = request.POST['action_id']
    except Exception:
        logger.error("Can't find action ID in request", )
        pass
    print "act_id:%s" % act_id
    try:
        actObj = get_object_or_404(Action, id=act_id)
        be = Backend()
        logger.info("Action: '%s'", actObj.name)
        be.exec_action(actObj)
    except Action.DoesNotExist:
        raise Http404
    logger.info("%s is executed", actObj.name)
    msg = "ok"
    return StreamingHttpResponse( msg)

 def __init__(self, db):
     # each TCP request handler gets its own sqlite cursor
     # this should be sufficient for proper concurrent behavior,
     # but if anyone can confirm this, that would be appreciated
     self.backend = Backend(db, exists=True)
     # support both \n and (default) \r\n delimiters
     # http://stackoverflow.com/questions/5087559/twisted-server-nc-client
     self.delimiter = "\n"

class Worker(Process):

	def __init__(self, backendUrl):
		super(Worker, self).__init__()
		self.name = socket.gethostname()
		self.backend = Backend(backendUrl)
		worker = self.backend.worker_for_name(self.name)
		if '_id' in worker:
			self.workerId = worker['_id']
			self.backend.workerId = self.workerId
		else:
			raise Exception('Worker has no id!!!!')
		self._stop = Event()

	def stop(self):
		self._stop.set()

	def stopped(self):
		return self._stop.is_set()

	def run(self):
 		deviceLoops = {}

		while not self.stopped():
			devices = iDevice.devices()
			currDeviceUDIDs = []
			# search for new devices
			for device in devices:
				currDeviceUDIDs.append(device.udid)
				if device.udid not in deviceLoops:
					dLoop = DeviceLoop(device, self.backend)
					dLoop.start()
					deviceLoops[device.udid] = dLoop
					logger.info('Started device loop for %s', device.udid)

			# cleanup finished processes
			for udid in deviceLoops.keys():
				if udid not in currDeviceUDIDs:
					dLoop = deviceLoops[udid]
					if dLoop.is_alive():
						dLoop.stop()
						logger.info('Waiting for DeviceLoop to stop... (%s)', udid)
						dLoop.join(10)
						if dLoop.is_alive():
							logger.info('... loop has not yet stopped. Terminating the loop now.  (%s)', udid)
							dLoop.terminate()
					deviceLoops.pop(udid)
					logger.info('Device loop finished: %s', udid)
			time.sleep(5)

		logger.info('runloop is shutting down. Stoping all client processes gracefully')
		for udid, process in deviceLoops:
			logger.info('joining device loop for device %s', udid)
			process.join()
		logger.info('Worker has finished working...')