def test_addressbookRevisionChangeConcurrency(self):
        """
        Test that two concurrent attempts to add resources in two separate
        calendar homes does not deadlock on the revision table update.
        """

        # Create first events in different addressbook homes
        txn1 = self.store.newTransaction()
        txn2 = self.store.newTransaction()

        addressbook_uid1_in_txn1 = yield self.addressbookUnderTest(txn1, "addressbook", "user01")
        addressbook_uid2_in_txn2 = yield self.addressbookUnderTest(txn2, "addressbook", "user02")

        data = """BEGIN:VCARD
VERSION:3.0
PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
UID:data%(ctr)s
FN:Data %(ctr)s
N:Sub Group;;;;
REV:20120503T194243Z
END:VCARD

"""

        component = Component.fromString(data % {"ctr": 1})
        yield addressbook_uid1_in_txn1.createAddressBookObjectWithName("data1.ics", component)

        component = Component.fromString(data % {"ctr": 2})
        yield addressbook_uid2_in_txn2.createAddressBookObjectWithName("data2.ics", component)

        # Setup deferreds to run concurrently and create second events in the calendar homes
        # previously used by the other transaction - this could create the deadlock.
        @inlineCallbacks
        def _defer_uid3():
            addressbook_uid1_in_txn2 = yield self.addressbookUnderTest(txn2, "addressbook", "user01")
            component = Component.fromString(data % {"ctr": 3})
            yield addressbook_uid1_in_txn2.createAddressBookObjectWithName("data3.ics", component)
            yield txn2.commit()
        d1 = _defer_uid3()

        @inlineCallbacks
        def _defer_uid4():
            addressbook_uid2_in_txn1 = yield self.addressbookUnderTest(txn1, "addressbook", "user02")
            component = Component.fromString(data % {"ctr": 4})
            yield addressbook_uid2_in_txn1.createAddressBookObjectWithName("data4.ics", component)
            yield txn1.commit()
        d2 = _defer_uid4()

        # Now do the concurrent provision attempt
        yield DeferredList([d1, d2])

        # Verify we did not have a deadlock and all resources have been created.
        vcarddata1 = yield self.addressbookObjectUnderTest(name="data1.ics", addressbook_name="addressbook", home="user01")
        vcarddata2 = yield self.addressbookObjectUnderTest(name="data2.ics", addressbook_name="addressbook", home="user02")
        vcarddata3 = yield self.addressbookObjectUnderTest(name="data3.ics", addressbook_name="addressbook", home="user01")
        vcarddata4 = yield self.addressbookObjectUnderTest(name="data4.ics", addressbook_name="addressbook", home="user02")
        self.assertNotEqual(vcarddata1, None)
        self.assertNotEqual(vcarddata2, None)
        self.assertNotEqual(vcarddata3, None)
        self.assertNotEqual(vcarddata4, None)

    def test_updateAfterRevisionCleanup(self):
        """
        Make sure L{AddressBookObject}'s can be updated or removed after revision cleanup
        removes their revision table entry..
        """
        person = """BEGIN:VCARD
VERSION:3.0
N:Thompson;Default1;;;
FN:Default1 Thompson
EMAIL;type=INTERNET;type=WORK;type=pref:[email protected]
TEL;type=WORK;type=pref:1-555-555-5555
TEL;type=CELL:1-444-444-4444
item1.ADR;type=WORK;type=pref:;;1245 Test;Sesame Street;California;11111;USA
item1.X-ABADR:us
UID:uid-person
X-ADDRESSBOOKSERVER-KIND:person
END:VCARD
"""
        group = """BEGIN:VCARD
VERSION:3.0
N:Group;Fancy;;;
FN:Fancy Group
UID:uid-group
X-ADDRESSBOOKSERVER-KIND:group
X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:uid-person
END:VCARD
"""
        group_update = """BEGIN:VCARD
VERSION:3.0
N:Group2;Fancy;;;
FN:Fancy Group2
UID:uid-group
X-ADDRESSBOOKSERVER-KIND:group
X-ADDRESSBOOKSERVER-MEMBER:urn:uuid:uid-person
END:VCARD
"""

        yield self.addressbookHomeUnderTest(name="user01")
        adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
        yield adbk.createAddressBookObjectWithName("person.vcf", VCard.fromString(person))
        yield adbk.createAddressBookObjectWithName("group.vcf", VCard.fromString(group))
        yield self.commit()

        # Remove the revision
        adbk = yield self.addressbookUnderTest(home="user01", name="addressbook")
        yield adbk.syncToken()
        yield self.transactionUnderTest().deleteRevisionsBefore(adbk._syncTokenRevision + 1)
        yield self.commit()

        # Update the object
        obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
        yield obj.setComponent(VCard.fromString(group_update))
        yield self.commit()

        obj = yield self.addressbookObjectUnderTest(name="group.vcf", addressbook_name="addressbook", home="user01")
        self.assertTrue(obj is not None)
        obj = yield self.addressbookObjectUnderTest(name="person.vcf", addressbook_name="addressbook", home="user01")
        self.assertTrue(obj is not None)
        yield self.commit()

    def propFilter(self, properties, vcard):
        """
        Returns a vCard component object filtered according to the properties.
        """

        result = Component("VCARD")

        xml_properties = properties

        # Empty element means do all properties and components
        if xml_properties is None:
            xml_properties = AllProperties()

        if xml_properties is not None:
            if xml_properties == AllProperties():
                for vcard_property in vcard.properties():
                    result.addProperty(vcard_property)
            else:
                for xml_property in xml_properties:
                    name = xml_property.property_name
                    for vcard_property in vcard.properties(name):
                        result.addProperty(vcard_property)

                # add required properties
                for requiredProperty in ('N', 'FN', 'VERSION'):
                    if not result.hasProperty(requiredProperty):
                        result.addProperty(vcard.getProperty(requiredProperty))

        return result

    def test_index_revisions(self):
        data1 = """BEGIN:VCARD
VERSION:3.0
PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
UID:12345-67890-1-1.1
FN:Cyrus Daboo
N:Daboo;Cyrus
EMAIL;TYPE=INTERNET,PREF:[email protected]
END:VCARD
"""
        data2 = """BEGIN:VCARD
VERSION:3.0
PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
UID:12345-67890-2-1.1
FN:Wilfredo Sanchez
N:Sanchez;Wilfredo
EMAIL;TYPE=INTERNET,PREF:[email protected]
END:VCARD
"""
        data3 = """BEGIN:VCARD
VERSION:3.0
PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
UID:12345-67890-3-1.1
FN:Bruce Gaya
N:Gaya;Bruce
EMAIL;TYPE=INTERNET,PREF:[email protected]
END:VCARD
"""

        vcard = Component.fromString(data1)
        self.db.addResource("data1.vcf", vcard)
        vcard = Component.fromString(data2)
        self.db.addResource("data2.vcf", vcard)
        vcard = Component.fromString(data3)
        self.db.addResource("data3.vcf", vcard)
        self.db.deleteResource("data3.vcf")

        tests = (
            (0, (["data1.vcf", "data2.vcf", ], [],)),
            (1, (["data2.vcf", ], ["data3.vcf", ],)),
            (2, ([], ["data3.vcf", ],)),
            (3, ([], ["data3.vcf", ],)),
            (4, ([], [],)),
            (5, ([], [],)),
        )

        for revision, results in tests:
            self.assertEquals(self.db.whatchanged(revision), results, "Mismatched results for whatchanged with revision %d" % (revision,))

    def test_setComponent(self):
        """
        L{AddressBookObject.setComponent} changes the result of
        L{AddressBookObject.component} within the same transaction.
        """
        component = VComponent.fromString(vcard1modified_text)

        addressbook1 = yield self.addressbookUnderTest()
        addressbookObject = yield addressbook1.addressbookObjectWithName("1.vcf")
        oldComponent = yield addressbookObject.component()
        self.assertNotEqual(component, oldComponent)
        yield addressbookObject.setComponent(component)
        self.assertEquals((yield addressbookObject.component()), component)

        # Also check a new instance
        addressbookObject = yield addressbook1.addressbookObjectWithName("1.vcf")
        self.assertEquals((yield addressbookObject.component()), component)

        # notify is called prior to commit
        self.assertEquals(
            set(self.notifierFactory.history),
            set([
                ("/CardDAV/example.com/home1/", PushPriority.high),
                ("/CardDAV/example.com/home1/addressbook/", PushPriority.high),
            ])
        )

        yield self.commit()

    def test_purgeUID(self):
        txn = self._sqlCalendarStore.newTransaction()

        # Create an addressbook and one CardDAV resource
        abHome = (yield txn.addressbookHomeWithUID("home1", create=True))
        abColl = (yield abHome.addressbookWithName("addressbook"))
        (yield abColl.createAddressBookObjectWithName("card1",
            VCardComponent.fromString(VCARD_1)))
        self.assertEquals(len((yield abColl.addressbookObjects())), 1)

        # Verify there are 8 events in calendar1
        calHome = (yield txn.calendarHomeWithUID("home1"))
        calColl = (yield calHome.calendarWithName("calendar1"))
        self.assertEquals(len((yield calColl.calendarObjects())), 8)

        # Make the newly created objects available to the purgeUID transaction
        (yield txn.commit())

        # Purge home1
        total, ignored = (yield PurgePrincipalService.purgeUIDs(self._sqlCalendarStore, self.directory,
            self.rootResource, ("home1",), verbose=False, proxies=False,
            when=PyCalendarDateTime(now, 4, 1, 12, 0, 0, 0, PyCalendarTimezone(utc=True))))

        # 4 items deleted: 3 events and 1 vcard
        self.assertEquals(total, 4)

        txn = self._sqlCalendarStore.newTransaction()
        # adressbook home is deleted since it's now empty
        abHome = (yield txn.addressbookHomeWithUID("home1"))
        self.assertEquals(abHome, None)

        calHome = (yield txn.calendarHomeWithUID("home1"))
        calColl = (yield calHome.calendarWithName("calendar1"))
        self.assertEquals(len((yield calColl.calendarObjects())), 5)

    def test_setComponent(self):
        """
        L{AddressBookObject.setComponent} changes the result of
        L{AddressBookObject.component} within the same transaction.
        """
        component = VComponent.fromString(vcard1modified_text)

        addressbook1 = yield self.addressbookUnderTest()
        addressbookObject = yield addressbook1.addressbookObjectWithName("1.vcf")
        oldComponent = yield addressbookObject.component()
        self.assertNotEqual(component, oldComponent)
        yield addressbookObject.setComponent(component)
        self.assertEquals((yield addressbookObject.component()), component)

        # Also check a new instance
        addressbookObject = yield addressbook1.addressbookObjectWithName("1.vcf")
        self.assertEquals((yield addressbookObject.component()), component)

        yield self.commit()

        # Make sure notification fired after commit
        self.assertEquals(
            self.notifierFactory.history,
            [
                "CardDAV|home1",
                "CardDAV|home1/addressbook_1",
            ]
        )

    def populateOneAddressBookObject(self, objectName, objectText):
        """
        Populate one addressbook object in the test user's addressbook.

        @param objectName: The name of a addressbook object.
        @type objectName: str
        @param objectText: Some iVcard text to populate it with.
        @type objectText: str
        """
        record = self.directoryService.recordWithShortName("users", "wsanchez")
        uid = record.uid
        # XXX there should be a more test-friendly way to ensure the directory
        # actually exists
        try:
            self.addressbookCollection._newStore._path.createDirectory()
        except:
            pass
        txn = self.addressbookCollection._newStore.newTransaction()
        home = yield txn.addressbookHomeWithUID(uid, True)
        adbk = yield home.addressbookWithName("addressbook")
        if adbk is None:
            yield home.createAddressBookWithName("addressbook")
            adbk = yield home.addressbookWithName("addressbook")
        yield adbk.createAddressBookObjectWithName(
            objectName, VCComponent.fromString(objectText)
        )
        yield txn.commit()

    def test_purgeUIDCompletely(self):
        txn = self._sqlCalendarStore.newTransaction()

        # Create an addressbook and one CardDAV resource
        abHome = (yield txn.addressbookHomeWithUID("home1", create=True))
        abColl = (yield abHome.addressbookWithName("addressbook"))
        (yield abColl.createAddressBookObjectWithName("card1",
            VCardComponent.fromString(VCARD_1)))
        self.assertEquals(len((yield abColl.addressbookObjects())), 1)

        # Verify there are 8 events in calendar1
        calHome = (yield txn.calendarHomeWithUID("home1"))
        calColl = (yield calHome.calendarWithName("calendar1"))
        self.assertEquals(len((yield calColl.calendarObjects())), 8)

        # Make the newly created objects available to the purgeUID transaction
        (yield txn.commit())

        # Purge home1 completely
        total, ignored = (yield PurgePrincipalService.purgeUIDs(self._sqlCalendarStore, self.directory,
            self.rootResource, ("home1",), verbose=False, proxies=False, completely=True))

        # 9 items deleted: 8 events and 1 vcard
        self.assertEquals(total, 9)

        # Homes have been deleted as well
        txn = self._sqlCalendarStore.newTransaction()
        abHome = (yield txn.addressbookHomeWithUID("home1"))
        self.assertEquals(abHome, None)
        calHome = (yield txn.calendarHomeWithUID("home1"))
        self.assertEquals(calHome, None)

def populateAddressBooksFrom(requirements, store):
    """
    Populate C{store} from C{requirements}.

    @param requirements: a dictionary of the format described by
        L{txdav.caldav.datastore.test.common.CommonTests.requirements}.

    @param store: the L{IDataStore} to populate with addressbook data.
    """
    populateTxn = store.newTransaction()
    for homeUID in requirements:
        addressbooks = requirements[homeUID]
        if addressbooks is not None:
            home = yield populateTxn.addressbookHomeWithUID(homeUID, True)
            # We don't want the default addressbook
            try:
                yield home.removeAddressBookWithName("addressbook")
            except NoSuchHomeChildError:
                pass
            for addressbookName in addressbooks:
                addressbookObjNames = addressbooks[addressbookName]
                if addressbookObjNames is not None:
                    # XXX should not be yielding!  this SQL will be executed
                    # first!
                    yield home.createAddressBookWithName(addressbookName)
                    addressbook = yield home.addressbookWithName(addressbookName)
                    for objectName in addressbookObjNames:
                        objData = addressbookObjNames[objectName]
                        yield addressbook.createAddressBookObjectWithName(
                            objectName,
                            ABComponent.fromString(objData),
                        )
    yield populateTxn.commit()

    def test_createAddressBookObjectWithName_absent(self):
        """
        L{IAddressBook.createAddressBookObjectWithName} creates a new
        L{IAddressBookObject}.
        """
        addressbook1 = yield self.addressbookUnderTest()
        name = "4.vcf"
        self.assertIdentical((yield addressbook1.addressbookObjectWithName(name)), None)
        component = VComponent.fromString(vcard4_text)
        yield addressbook1.createAddressBookObjectWithName(name, component)

        addressbookObject = yield addressbook1.addressbookObjectWithName(name)
        self.assertEquals((yield addressbookObject.component()), component)

        # notify is called prior to commit
        self.assertEquals(
            set(self.notifierFactory.history),
            set(
                [
                    ("/CardDAV/example.com/home1/", PushPriority.high),
                    ("/CardDAV/example.com/home1/addressbook/", PushPriority.high),
                ]
            ),
        )

        yield self.commit()

    def _db_recreate(self, do_commit=True):
        """
        Re-create the database tables from existing address book data.
        """

        #
        # Populate the DB with data from already existing resources.
        # This allows for index recovery if the DB file gets
        # deleted.
        #
        fp = self.resource.fp
        for name in fp.listdir():
            if name.startswith("."):
                continue

            try:
                stream = fp.child(name).open()
            except (IOError, OSError), e:
                log.error("Unable to open resource %s: %s" % (name, e))
                continue

            try:
                # FIXME: This is blocking I/O
                try:
                    vcard = Component.fromStream(stream)
                    vcard.validVCardData()
                    vcard.validForCardDAV()
                except ValueError:
                    log.error("Non-addressbook resource: %s" % (name,))
                else:
                    #log.info("Indexing resource: %s" % (name,))
                    self.addResource(name, vcard, True)
            finally:
                stream.close()

    def test_index(self):
        data = (
            (
                "#1.1 Simple component",
                "1.1",
                """BEGIN:VCARD
VERSION:3.0
PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
UID:12345-67890-1.1
FN:Cyrus Daboo
N:Daboo;Cyrus
EMAIL;TYPE=INTERNET,PREF:[email protected]
END:VCARD
""",
            ),
        )

        for description, name, vcard_txt in data:
            calendar = Component.fromString(vcard_txt)
            f = open(os.path.join(self.site.resource.fp.path, name), "w")
            f.write(vcard_txt)
            del f

            self.db.addResource(name, calendar)
            self.assertTrue(self.db.resourceExists(name), msg=description)

        self.db._db_recreate()
        for description, name, vcard_txt in data:
            self.assertTrue(self.db.resourceExists(name), msg=description)

    def test_setComponentPreservesProperties(self):
        """
        L{IAddressBookObject.setComponent} preserves properties.

        (Some implementations must go to extra trouble to provide this
        behavior; for example, file storage must copy extended attributes from
        the existing file to the temporary file replacing it.)
        """
        propertyName = PropertyName("http://example.com/ns", "example")
        propertyContent = WebDAVUnknownElement("sample content")
        propertyContent.name = propertyName.name
        propertyContent.namespace = propertyName.namespace

        abobject = (yield self.addressbookObjectUnderTest())
        if abobject._parentCollection.objectResourcesHaveProperties():
            (yield self.addressbookObjectUnderTest()).properties()[propertyName] = propertyContent
            yield self.commit()
            # Sanity check; are properties even readable in a separate transaction?
            # Should probably be a separate test.
            self.assertEquals((yield self.addressbookObjectUnderTest()).properties()[propertyName], propertyContent)
            obj = yield self.addressbookObjectUnderTest()
            vcard1_text = yield obj._text()
            vcard1_text_withDifferentNote = vcard1_text.replace("NOTE:CardDAV protocol updates", "NOTE:Changed")
            # Sanity check; make sure the test has the right idea of the subject.
            self.assertNotEquals(vcard1_text, vcard1_text_withDifferentNote)
            newComponent = VComponent.fromString(vcard1_text_withDifferentNote)
            yield obj.setComponent(newComponent)

            # Putting everything into a separate transaction to account for any
            # caching that may take place.
            yield self.commit()
            self.assertEquals((yield self.addressbookObjectUnderTest()).properties()[propertyName], propertyContent)

    def fullValidation(self):
        """
        Do full validation of source and destination vcard data.
        """

        if self.destinationadbk:
            # Valid resource name check
            result, message = self.validResourceName()
            if not result:
                log.err(message)
                raise HTTPError(StatusResponse(responsecode.FORBIDDEN, "Resource name not allowed"))

            if not self.sourceadbk:
                # Valid content type check on the source resource if its not in a vcard collection
                if self.source is not None:
                    result, message = self.validContentType()
                    if not result:
                        log.err(message)
                        raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (carddav_namespace, "supported-address-data")))
                
                    # At this point we need the calendar data to do more tests
                    self.vcard = self.source.vCard()
                else:
                    try:
                        if type(self.vcard) in (types.StringType, types.UnicodeType,):
                            self.vcarddata = self.vcard
                            self.vcard = Component.fromString(self.vcard)
                    except ValueError, e:
                        log.err(str(e))
                        raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (carddav_namespace, "valid-address-data")))
                        
                # Valid vcard data for CalDAV check
                result, message = self.validAddressDataCheck()
                if not result:
                    log.err(message)
                    raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (carddav_namespace, "valid-addressbook-object-resource")))

                # Must have a valid UID at this point
                self.uid = self.vcard.resourceUID()
            else:
                # Get UID from original resource
                self.source_index = self.sourceparent.index()
                self.uid = self.source_index.resourceUIDForName(self.source.name())
                if self.uid is None:
                    log.err("Source vcard does not have a UID: %s" % self.source.name())
                    raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (carddav_namespace, "valid-addressbook-object-resource")))

                # FIXME: We need this here because we have to re-index the destination. Ideally it
                # would be better to copy the index entries from the source and add to the destination.
                self.vcard = self.source.vCard()

            # Valid vcard data size check
            result, message = self.validSizeCheck()
            if not result:
                log.err(message)
                raise HTTPError(ErrorResponse(responsecode.FORBIDDEN, (carddav_namespace, "max-resource-size")))

            # Check access
            return succeed(None)

 def test_newAddressBookObjectProperties(self):
     """
     L{IAddressBookObject.properties} returns an empty property store for a
     addressbook object which has been created but not committed.
     """
     addressbook = yield self.addressbookUnderTest()
     yield addressbook.createAddressBookObjectWithName("4.vcf", VComponent.fromString(vcard4_text))
     newEvent = yield addressbook.addressbookObjectWithName("4.vcf")
     self.assertEquals(newEvent.properties().items(), [])

 def address(self):
     """
     Returns an address component derived from this element.
     """
     data = self.addressData()
     if data:
         return Component.fromString(data, format=self.content_type)
     else:
         return None

 def test_setComponent_invalid(self):
     """
     L{IAddressBookObject.setComponent} raises L{InvalidIAddressBookDataError} if
     presented with invalid iAddressBook text.
     """
     addressbookObject = (yield self.addressbookObjectUnderTest())
     yield self.failUnlessFailure(
         maybeDeferred(addressbookObject.setComponent, VComponent.fromString(vcard4notCardDAV_text)),
         InvalidObjectResourceError,
     )

 def test_setComponent_invalid(self):
     """
     L{IAddressBookObject.setComponent} raises L{InvalidIAddressBookDataError} if
     presented with invalid iAddressBook text.
     """
     addressbookObject = self.addressbookObjectUnderTest()
     self.assertRaises(
         InvalidObjectResourceError,
         addressbookObject.setComponent,
         VComponent.fromString(vcard4notCardDAV_text)
     )

 def test_createAddressBookObjectWithName_exists(self):
     """
     L{IAddressBook.createAddressBookObjectWithName} raises
     L{AddressBookObjectNameAlreadyExistsError} if a addressbook object with the
     given name already exists in that addressbook.
     """
     self.assertRaises(
         ObjectResourceNameAlreadyExistsError,
         self.addressbookUnderTest().createAddressBookObjectWithName,
         "1.vcf", VComponent.fromString(vcard4_text)
     )