Skip to content

Commit

Permalink
Add support for the PCI(e)-PCI bridges to the Intel VT-d driver. The
Browse files Browse the repository at this point in the history 
bridge takes ownership of the transaction, so bsf of the requester is
the bridge and not a device behind it.  As result, code needs to walk
the hierarchy up to use correct context.

Note that PCIe->PCI-X bridges are not handled quite correctly since
such bridges are allowed to only take ownership of some transactions.
Also, weird but unrealistic cases of PCIe behind PCI bus are also not
handled.

Still, the patch provides significant step forward for the bridge
handling.

Submitted by:	Jason Harmening <[email protected]>
MFC after:	1 week
  • Loading branch information
@kostikbel
kostikbel committed Mar 18, 2014
1 parent f8f1450 commit 97d7557
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 17 deletions.
122 changes: 113 additions & 9 deletions sys/x86/iommu/busdma_dmar.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
#include <sys/taskqueue.h>
#include <sys/tree.h>
#include <sys/uio.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
Expand All @@ -69,15 +70,10 @@ __FBSDID("$FreeBSD$");
*/

static bool
dmar_bus_dma_is_dev_disabled(device_t dev)
dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
{
char str[128], *env;
int domain, bus, slot, func;

domain = pci_get_domain(dev);
bus = pci_get_bus(dev);
slot = pci_get_slot(dev);
func = pci_get_function(dev);
snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d.bounce",
domain, bus, slot, func);
env = getenv(str);
Expand All @@ -87,11 +83,119 @@ dmar_bus_dma_is_dev_disabled(device_t dev)
return (true);
}

/*
* Given original device, find the requester ID that will be seen by
* the DMAR unit and used for page table lookup. PCI bridges may take
* ownership of transactions from downstream devices, so it may not be
* the same as the BSF of the target device. In those cases, all
* devices downstream of the bridge must share a single mapping
* domain, and must collectively be assigned to use either DMAR or
* bounce mapping.
*/
static device_t
dmar_get_requester(device_t dev, int *bus, int *slot, int *func)
{
devclass_t pci_class;
device_t pci, pcib, requester;
int cap_offset;

pci_class = devclass_find("pci");
requester = dev;

*bus = pci_get_bus(dev);
*slot = pci_get_slot(dev);
*func = pci_get_function(dev);

/*
* Walk the bridge hierarchy from the target device to the
* host port to find the translating bridge nearest the DMAR
* unit.
*/
for (;;) {
pci = device_get_parent(dev);
KASSERT(pci != NULL, ("NULL parent for pci%d:%d:%d:%d",
pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
pci_get_function(dev)));
KASSERT(device_get_devclass(pci) == pci_class,
("Non-pci parent for pci%d:%d:%d:%d",
pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
pci_get_function(dev)));

pcib = device_get_parent(pci);
KASSERT(pcib != NULL, ("NULL bridge for pci%d:%d:%d:%d",
pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
pci_get_function(dev)));

/*
* The parent of our "bridge" isn't another PCI bus,
* so pcib isn't a PCI->PCI bridge but rather a host
* port, and the requester ID won't be translated
* further.
*/
if (device_get_devclass(device_get_parent(pcib)) != pci_class)
break;

if (pci_find_cap(dev, PCIY_EXPRESS, &cap_offset) != 0) {
/*
* Device is not PCIe, it cannot be seen as a
* requester by DMAR unit.
*/
requester = pcib;

/* Check whether the bus above is PCIe. */
if (pci_find_cap(pcib, PCIY_EXPRESS,
&cap_offset) == 0) {
/*
* The current device is not PCIe, but
* the bridge above it is. This is a
* PCIe->PCI bridge. Assume that the
* requester ID will be the secondary
* bus number with slot and function
* set to zero.
*
* XXX: Doesn't handle the case where
* the bridge is PCIe->PCI-X, and the
* bridge will only take ownership of
* requests in some cases. We should
* provide context entries with the
* same page tables for taken and
* non-taken transactions.
*/
*bus = pci_get_bus(dev);
*slot = *func = 0;
} else {
/*
* Neither the device nor the bridge
* above it are PCIe. This is a
* conventional PCI->PCI bridge, which
* will use the bridge's BSF as the
* requester ID.
*/
*bus = pci_get_bus(pcib);
*slot = pci_get_slot(pcib);
*func = pci_get_function(pcib);
}
}
/*
* Do not stop the loop even if the target device is
* PCIe, because it is possible (but unlikely) to have
* a PCI->PCIe bridge somewhere in the hierarchy.
*/

dev = pcib;
}
return (requester);
}

struct dmar_ctx *
dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
{
device_t requester;
struct dmar_ctx *ctx;
bool disabled;
int bus, slot, func;

requester = dmar_get_requester(dev, &bus, &slot, &func);

/*
* If the user requested the IOMMU disabled for the device, we
Expand All @@ -100,11 +204,11 @@ dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
* Instead provide the identity mapping for the device
* context.
*/
disabled = dmar_bus_dma_is_dev_disabled(dev);
ctx = dmar_get_ctx(dmar, dev, disabled, rmrr);
disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(dev), bus,
slot, func);
ctx = dmar_get_ctx(dmar, requester, bus, slot, func, disabled, rmrr);
if (ctx == NULL)
return (NULL);
ctx->ctx_tag.owner = dev;
if (disabled) {
/*
* Keep the first reference on context, release the
Expand Down
15 changes: 8 additions & 7 deletions sys/x86/iommu/intel_ctx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,17 +262,15 @@ dmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited)
}

struct dmar_ctx *
dmar_get_ctx(struct dmar_unit *dmar, device_t dev, bool id_mapped, bool rmrr_init)
dmar_get_ctx(struct dmar_unit *dmar, device_t dev, int bus, int slot, int func,
bool id_mapped, bool rmrr_init)
{
struct dmar_ctx *ctx, *ctx1;
dmar_ctx_entry_t *ctxp;
struct sf_buf *sf;
int bus, slot, func, error, mgaw;
int error, mgaw;
bool enable;

bus = pci_get_bus(dev);
slot = pci_get_slot(dev);
func = pci_get_function(dev);
enable = false;
TD_PREP_PINNED_ASSERT;
DMAR_LOCK(dmar);
Expand Down Expand Up @@ -356,6 +354,7 @@ dmar_get_ctx(struct dmar_unit *dmar, device_t dev, bool id_mapped, bool rmrr_ini
ctx = dmar_find_ctx_locked(dmar, bus, slot, func);
if (ctx == NULL) {
ctx = ctx1;
ctx->ctx_tag.owner = dev;
ctx->domain = alloc_unrl(dmar->domids);
if (ctx->domain == -1) {
DMAR_UNLOCK(dmar);
Expand All @@ -376,9 +375,11 @@ dmar_get_ctx(struct dmar_unit *dmar, device_t dev, bool id_mapped, bool rmrr_ini
LIST_INSERT_HEAD(&dmar->contexts, ctx, link);
ctx_id_entry_init(ctx, ctxp);
device_printf(dev,
"dmar%d pci%d:%d:%d:%d domain %d mgaw %d agaw %d\n",
"dmar%d pci%d:%d:%d:%d domain %d mgaw %d "
"agaw %d %s-mapped\n",
dmar->unit, dmar->segment, bus, slot,
func, ctx->domain, ctx->mgaw, ctx->agaw);
func, ctx->domain, ctx->mgaw, ctx->agaw,
id_mapped ? "id" : "re");
} else {
dmar_ctx_dtr(ctx1, true, true);
}
Expand Down
2 changes: 1 addition & 1 deletion sys/x86/iommu/intel_dmar.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ void ctx_free_pgtbl(struct dmar_ctx *ctx);
struct dmar_ctx *dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev,
bool rmrr);
struct dmar_ctx *dmar_get_ctx(struct dmar_unit *dmar, device_t dev,
bool id_mapped, bool rmrr_init);
int bus, int slot, int func, bool id_mapped, bool rmrr_init);
void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
void dmar_free_ctx(struct dmar_ctx *ctx);
struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, int bus,
Expand Down

0 comments on commit 97d7557

Please sign in to comment.