commit 744086710956c59fa06bbc3cfde075f616469cb1
Author: Daniel P. Berrange <berrange@redhat.com>
Date:   Wed Jan 22 17:28:29 2014 +0000

    Push nwfilter update locking up to top level
    
    The NWFilter code has as a deadlock race condition between
    the virNWFilter{Define,Undefine} APIs and starting of guest
    VMs due to mis-matched lock ordering.
    
    In the virNWFilter{Define,Undefine} codepaths the lock ordering
    is
    
      1. nwfilter driver lock
      2. virt driver lock
      3. nwfilter update lock
      4. domain object lock
    
    In the VM guest startup paths the lock ordering is
    
      1. virt driver lock
      2. domain object lock
      3. nwfilter update lock
    
    As can be seen the domain object and nwfilter update locks are
    not acquired in a consistent order.
    
    The fix used is to push the nwfilter update lock upto the top
    level resulting in a lock ordering for virNWFilter{Define,Undefine}
    of
    
      1. nwfilter driver lock
      2. nwfilter update lock
      3. virt driver lock
      4. domain object lock
    
    and VM start using
    
      1. nwfilter update lock
      2. virt driver lock
      3. domain object lock
    
    This has the effect of serializing VM startup once again, even if
    no nwfilters are applied to the guest. There is also the possibility
    of deadlock due to a call graph loop via virNWFilterInstantiate
    and virNWFilterInstantiateFilterLate.
    
    These two problems mean the lock must be turned into a read/write
    lock instead of a plain mutex at the same time. The lock is used to
    serialize changes to the "driver->nwfilters" hash, so the write lock
    only needs to be held by the define/undefine methods. All other
    methods can rely on a read lock which allows good concurrency.
    
    Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
    (cherry picked from commit 6e5c79a1b5a8b3a23e7df7ffe58fb272aa17fbfb)

Index: libvirt-1.1.1/src/conf/nwfilter_conf.c
===================================================================
--- libvirt-1.1.1.orig/src/conf/nwfilter_conf.c	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/conf/nwfilter_conf.c	2014-03-25 14:15:26.914627547 -0500
@@ -143,17 +143,22 @@ static const struct int_map chain_priori
 /*
  * only one filter update allowed
  */
-static virMutex updateMutex;
+static virRWLock updateLock;
 static bool initialized = false;
 
 void
-virNWFilterLockFilterUpdates(void) {
-    virMutexLock(&updateMutex);
+virNWFilterReadLockFilterUpdates(void) {
+    virRWLockRead(&updateLock);
+}
+
+void
+virNWFilterWriteLockFilterUpdates(void) {
+    virRWLockWrite(&updateLock);
 }
 
 void
 virNWFilterUnlockFilterUpdates(void) {
-    virMutexUnlock(&updateMutex);
+    virRWLockUnlock(&updateLock);
 }
 
 
@@ -2981,14 +2986,12 @@ virNWFilterObjAssignDef(virNWFilterObjLi
         return NULL;
     }
 
-    virNWFilterLockFilterUpdates();
 
     if ((nwfilter = virNWFilterObjFindByName(nwfilters, def->name))) {
 
         if (virNWFilterDefEqual(def, nwfilter->def, false)) {
             virNWFilterDefFree(nwfilter->def);
             nwfilter->def = def;
-            virNWFilterUnlockFilterUpdates();
             return nwfilter;
         }
 
@@ -2996,7 +2999,6 @@ virNWFilterObjAssignDef(virNWFilterObjLi
         /* trigger the update on VMs referencing the filter */
         if (virNWFilterTriggerVMFilterRebuild()) {
             nwfilter->newDef = NULL;
-            virNWFilterUnlockFilterUpdates();
             virNWFilterObjUnlock(nwfilter);
             return NULL;
         }
@@ -3004,12 +3006,9 @@ virNWFilterObjAssignDef(virNWFilterObjLi
         virNWFilterDefFree(nwfilter->def);
         nwfilter->def = def;
         nwfilter->newDef = NULL;
-        virNWFilterUnlockFilterUpdates();
         return nwfilter;
     }
 
-    virNWFilterUnlockFilterUpdates();
-
     if (VIR_ALLOC(nwfilter) < 0)
         return NULL;
 
@@ -3474,7 +3473,7 @@ int virNWFilterConfLayerInit(virDomainOb
 
     initialized = true;
 
-    if (virMutexInitRecursive(&updateMutex) < 0)
+    if (virRWLockInit(&updateLock) < 0)
         return -1;
 
     return 0;
@@ -3486,7 +3485,7 @@ void virNWFilterConfLayerShutdown(void)
     if (!initialized)
         return;
 
-    virMutexDestroy(&updateMutex);
+    virRWLockDestroy(&updateLock);
 
     initialized = false;
     virNWFilterDomainFWUpdateOpaque = NULL;
Index: libvirt-1.1.1/src/conf/nwfilter_conf.h
===================================================================
--- libvirt-1.1.1.orig/src/conf/nwfilter_conf.h	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/conf/nwfilter_conf.h	2014-03-25 14:15:26.914627547 -0500
@@ -716,7 +716,8 @@ virNWFilterDefPtr virNWFilterDefParseFil
 void virNWFilterObjLock(virNWFilterObjPtr obj);
 void virNWFilterObjUnlock(virNWFilterObjPtr obj);
 
-void virNWFilterLockFilterUpdates(void);
+void virNWFilterWriteLockFilterUpdates(void);
+void virNWFilterReadLockFilterUpdates(void);
 void virNWFilterUnlockFilterUpdates(void);
 
 int virNWFilterConfLayerInit(virDomainObjListIterator domUpdateCB, void *opaque);
Index: libvirt-1.1.1/src/libvirt_private.syms
===================================================================
--- libvirt-1.1.1.orig/src/libvirt_private.syms	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/libvirt_private.syms	2014-03-25 14:15:26.914627547 -0500
@@ -564,7 +564,6 @@ virNWFilterDefParseString;
 virNWFilterInstFiltersOnAllVMs;
 virNWFilterJumpTargetTypeToString;
 virNWFilterLoadAllConfigs;
-virNWFilterLockFilterUpdates;
 virNWFilterObjAssignDef;
 virNWFilterObjDeleteDef;
 virNWFilterObjFindByName;
@@ -576,6 +575,7 @@ virNWFilterObjSaveDef;
 virNWFilterObjUnlock;
 virNWFilterPrintStateMatchFlags;
 virNWFilterPrintTCPFlags;
+virNWFilterReadLockFilterUpdates;
 virNWFilterRegisterCallbackDriver;
 virNWFilterRuleActionTypeToString;
 virNWFilterRuleDirectionTypeToString;
@@ -583,6 +583,7 @@ virNWFilterRuleProtocolTypeToString;
 virNWFilterTestUnassignDef;
 virNWFilterUnlockFilterUpdates;
 virNWFilterUnRegisterCallbackDriver;
+virNWFilterWriteLockFilterUpdates;
 
 
 # conf/nwfilter_ipaddrmap.h
Index: libvirt-1.1.1/src/lxc/lxc_driver.c
===================================================================
--- libvirt-1.1.1.orig/src/lxc/lxc_driver.c	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/lxc/lxc_driver.c	2014-03-25 14:15:26.918627547 -0500
@@ -935,6 +935,8 @@ static int lxcDomainCreateWithFiles(virD
 
     virCheckFlags(VIR_DOMAIN_START_AUTODESTROY, -1);
 
+    virNWFilterReadLockFilterUpdates();
+
     if (!(vm = lxcDomObjFromDomain(dom)))
         goto cleanup;
 
@@ -973,6 +975,7 @@ cleanup:
     if (event)
         virDomainEventStateQueue(driver->domainEventState, event);
     virObjectUnref(cfg);
+    virNWFilterUnlockFilterUpdates();
     return ret;
 }
 
@@ -1029,6 +1032,8 @@ lxcDomainCreateXMLWithFiles(virConnectPt
 
     virCheckFlags(VIR_DOMAIN_START_AUTODESTROY, NULL);
 
+    virNWFilterReadLockFilterUpdates();
+
     if (!(caps = virLXCDriverGetCapabilities(driver, false)))
         goto cleanup;
 
@@ -1084,6 +1089,7 @@ cleanup:
         virDomainEventStateQueue(driver->domainEventState, event);
     virObjectUnref(caps);
     virObjectUnref(cfg);
+    virNWFilterUnlockFilterUpdates();
     return dom;
 }
 
Index: libvirt-1.1.1/src/nwfilter/nwfilter_driver.c
===================================================================
--- libvirt-1.1.1.orig/src/nwfilter/nwfilter_driver.c	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/nwfilter/nwfilter_driver.c	2014-03-25 14:15:26.918627547 -0500
@@ -279,12 +279,14 @@ nwfilterStateReload(void)
     virNWFilterLearnThreadsTerminate(true);
 
     nwfilterDriverLock(driverState);
+    virNWFilterWriteLockFilterUpdates();
     virNWFilterCallbackDriversLock();
 
     virNWFilterLoadAllConfigs(&driverState->nwfilters,
                               driverState->configDir);
 
     virNWFilterCallbackDriversUnlock();
+    virNWFilterUnlockFilterUpdates();
     nwfilterDriverUnlock(driverState);
 
     virNWFilterInstFiltersOnAllVMs();
@@ -550,6 +552,7 @@ nwfilterDefineXML(virConnectPtr conn,
     virNWFilterPtr ret = NULL;
 
     nwfilterDriverLock(driver);
+    virNWFilterWriteLockFilterUpdates();
     virNWFilterCallbackDriversLock();
 
     if (!(def = virNWFilterDefParseString(xml)))
@@ -576,6 +579,7 @@ cleanup:
         virNWFilterObjUnlock(nwfilter);
 
     virNWFilterCallbackDriversUnlock();
+    virNWFilterUnlockFilterUpdates();
     nwfilterDriverUnlock(driver);
     return ret;
 }
@@ -588,10 +592,9 @@ nwfilterUndefine(virNWFilterPtr obj) {
     int ret = -1;
 
     nwfilterDriverLock(driver);
+    virNWFilterWriteLockFilterUpdates();
     virNWFilterCallbackDriversLock();
 
-    virNWFilterLockFilterUpdates();
-
     nwfilter = virNWFilterObjFindByUUID(&driver->nwfilters, obj->uuid);
     if (!nwfilter) {
         virReportError(VIR_ERR_NO_NWFILTER,
@@ -622,9 +625,8 @@ cleanup:
     if (nwfilter)
         virNWFilterObjUnlock(nwfilter);
 
-    virNWFilterUnlockFilterUpdates();
-
     virNWFilterCallbackDriversUnlock();
+    virNWFilterUnlockFilterUpdates();
     nwfilterDriverUnlock(driver);
     return ret;
 }
Index: libvirt-1.1.1/src/nwfilter/nwfilter_gentech_driver.c
===================================================================
--- libvirt-1.1.1.orig/src/nwfilter/nwfilter_gentech_driver.c	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/nwfilter/nwfilter_gentech_driver.c	2014-03-25 14:15:26.918627547 -0500
@@ -934,8 +934,6 @@ _virNWFilterInstantiateFilter(virNWFilte
     int ifindex;
     int rc;
 
-    virNWFilterLockFilterUpdates();
-
     /* after grabbing the filter update lock check for the interface; if
        it's not there anymore its filters will be or are being removed
        (while holding the lock) and we don't want to build new ones */
@@ -963,8 +961,6 @@ _virNWFilterInstantiateFilter(virNWFilte
                                         foundNewFilter);
 
 cleanup:
-    virNWFilterUnlockFilterUpdates();
-
     return rc;
 }
 
@@ -983,7 +979,7 @@ virNWFilterInstantiateFilterLate(virNWFi
     int rc;
     bool foundNewFilter = false;
 
-    virNWFilterLockFilterUpdates();
+    virNWFilterReadLockFilterUpdates();
 
     rc = __virNWFilterInstantiateFilter(driver,
                                         vmuuid,
Index: libvirt-1.1.1/src/qemu/qemu_driver.c
===================================================================
--- libvirt-1.1.1.orig/src/qemu/qemu_driver.c	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/qemu/qemu_driver.c	2014-03-25 14:15:26.926627547 -0500
@@ -1570,6 +1570,8 @@ static virDomainPtr qemuDomainCreateXML(
     if (flags & VIR_DOMAIN_START_AUTODESTROY)
         start_flags |= VIR_QEMU_PROCESS_START_AUTODESTROY;
 
+    virNWFilterReadLockFilterUpdates();
+
     if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
         goto cleanup;
 
@@ -1647,6 +1649,7 @@ cleanup:
     }
     virObjectUnref(caps);
     virObjectUnref(qemuCaps);
+    virNWFilterUnlockFilterUpdates();
     return dom;
 }
 
@@ -6101,6 +6104,8 @@ qemuDomainCreateWithFlags(virDomainPtr d
                   VIR_DOMAIN_START_BYPASS_CACHE |
                   VIR_DOMAIN_START_FORCE_BOOT, -1);
 
+    virNWFilterReadLockFilterUpdates();
+
     if (!(vm = qemuDomObjFromDomain(dom)))
         return -1;
 
@@ -6128,6 +6133,7 @@ endjob:
 cleanup:
     if (vm)
         virObjectUnlock(vm);
+    virNWFilterUnlockFilterUpdates();
     return ret;
 }
 
Index: libvirt-1.1.1/src/uml/uml_driver.c
===================================================================
--- libvirt-1.1.1.orig/src/uml/uml_driver.c	2014-03-25 14:15:26.930627547 -0500
+++ libvirt-1.1.1/src/uml/uml_driver.c	2014-03-25 14:15:26.926627547 -0500
@@ -1574,6 +1574,7 @@ static virDomainPtr umlDomainCreateXML(v
 
     virCheckFlags(VIR_DOMAIN_START_AUTODESTROY, NULL);
 
+    virNWFilterReadLockFilterUpdates();
     umlDriverLock(driver);
     if (!(def = virDomainDefParseString(xml, driver->caps, driver->xmlopt,
                                         1 << VIR_DOMAIN_VIRT_UML,
@@ -1613,6 +1614,7 @@ cleanup:
     if (event)
         umlDomainEventQueue(driver, event);
     umlDriverUnlock(driver);
+    virNWFilterUnlockFilterUpdates();
     return dom;
 }
 
@@ -1997,6 +1999,7 @@ static int umlDomainCreateWithFlags(virD
 
     virCheckFlags(VIR_DOMAIN_START_AUTODESTROY, -1);
 
+    virNWFilterReadLockFilterUpdates();
     umlDriverLock(driver);
     vm = virDomainObjListFindByUUID(driver->domains, dom->uuid);
 
@@ -2023,6 +2026,7 @@ cleanup:
     if (event)
         umlDomainEventQueue(driver, event);
     umlDriverUnlock(driver);
+    virNWFilterUnlockFilterUpdates();
     return ret;
 }
 
