Skip to content

Commit 815b33f

Browse files
joergroedeljoergroedel
committed
x86/amd-iommu: Cleanup completion-wait handling
This patch cleans up the implementation of completion-wait command sending. It also switches the completion indicator from the MMIO bit to a memory store which can be checked without IOMMU locking. As a side effect this patch makes the __iommu_queue_command function obsolete and so it is removed too. Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
1 parent 11b6402 commit 815b33f

File tree

1 file changed

+28
-79
lines changed

1 file changed

+28
-79
lines changed

‎arch/x86/kernel/amd_iommu.c

Lines changed: 28 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <linux/dma-mapping.h>
2626
#include <linux/iommu-helper.h>
2727
#include <linux/iommu.h>
28+
#include <linux/delay.h>
2829
#include <asm/proto.h>
2930
#include <asm/iommu.h>
3031
#include <asm/gart.h>
@@ -34,7 +35,7 @@
3435

3536
#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
3637

37-
#define EXIT_LOOP_COUNT 10000000
38+
#define LOOP_TIMEOUT 100000
3839

3940
static DEFINE_RWLOCK(amd_iommu_devtable_lock);
4041

@@ -383,10 +384,14 @@ irqreturn_t amd_iommu_int_handler(int irq, void *data)
383384
*
384385
****************************************************************************/
385386

386-
static void build_completion_wait(struct iommu_cmd *cmd)
387+
static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
387388
{
389+
WARN_ON(address & 0x7ULL);
390+
388391
memset(cmd, 0, sizeof(*cmd));
389-
cmd->data[0] = CMD_COMPL_WAIT_INT_MASK;
392+
cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
393+
cmd->data[1] = upper_32_bits(__pa(address));
394+
cmd->data[2] = 1;
390395
CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
391396
}
392397

@@ -432,12 +437,14 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
432437
* Writes the command to the IOMMUs command buffer and informs the
433438
* hardware about the new command. Must be called with iommu->lock held.
434439
*/
435-
static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
440+
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
436441
{
442+
unsigned long flags;
437443
u32 tail, head;
438444
u8 *target;
439445

440446
WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
447+
spin_lock_irqsave(&iommu->lock, flags);
441448
tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
442449
target = iommu->cmd_buf + tail;
443450
memcpy_toio(target, cmd, sizeof(*cmd));
@@ -446,99 +453,41 @@ static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
446453
if (tail == head)
447454
return -ENOMEM;
448455
writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
449-
450-
return 0;
451-
}
452-
453-
/*
454-
* General queuing function for commands. Takes iommu->lock and calls
455-
* __iommu_queue_command().
456-
*/
457-
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
458-
{
459-
unsigned long flags;
460-
int ret;
461-
462-
spin_lock_irqsave(&iommu->lock, flags);
463-
ret = __iommu_queue_command(iommu, cmd);
464-
if (!ret)
465-
iommu->need_sync = true;
456+
iommu->need_sync = true;
466457
spin_unlock_irqrestore(&iommu->lock, flags);
467458

468-
return ret;
469-
}
470-
471-
/*
472-
* This function waits until an IOMMU has completed a completion
473-
* wait command
474-
*/
475-
static void __iommu_wait_for_completion(struct amd_iommu *iommu)
476-
{
477-
int ready = 0;
478-
unsigned status = 0;
479-
unsigned long i = 0;
480-
481-
INC_STATS_COUNTER(compl_wait);
482-
483-
while (!ready && (i < EXIT_LOOP_COUNT)) {
484-
++i;
485-
/* wait for the bit to become one */
486-
status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
487-
ready = status & MMIO_STATUS_COM_WAIT_INT_MASK;
488-
}
489-
490-
/* set bit back to zero */
491-
status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
492-
writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
493-
494-
if (unlikely(i == EXIT_LOOP_COUNT))
495-
iommu->reset_in_progress = true;
459+
return 0;
496460
}
497461

498462
/*
499463
* This function queues a completion wait command into the command
500464
* buffer of an IOMMU
501465
*/
502-
static int __iommu_completion_wait(struct amd_iommu *iommu)
503-
{
504-
struct iommu_cmd cmd;
505-
506-
build_completion_wait(&cmd);
507-
508-
return __iommu_queue_command(iommu, &cmd);
509-
}
510-
511-
/*
512-
* This function is called whenever we need to ensure that the IOMMU has
513-
* completed execution of all commands we sent. It sends a
514-
* COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
515-
* us about that by writing a value to a physical address we pass with
516-
* the command.
517-
*/
518466
static int iommu_completion_wait(struct amd_iommu *iommu)
519467
{
520-
int ret = 0;
521-
unsigned long flags;
522-
523-
spin_lock_irqsave(&iommu->lock, flags);
468+
struct iommu_cmd cmd;
469+
volatile u64 sem = 0;
470+
int ret, i = 0;
524471

525472
if (!iommu->need_sync)
526-
goto out;
527-
528-
ret = __iommu_completion_wait(iommu);
473+
return 0;
529474

530-
iommu->need_sync = false;
475+
build_completion_wait(&cmd, (u64)&sem);
531476

477+
ret = iommu_queue_command(iommu, &cmd);
532478
if (ret)
533-
goto out;
534-
535-
__iommu_wait_for_completion(iommu);
479+
return ret;
536480

537-
out:
538-
spin_unlock_irqrestore(&iommu->lock, flags);
481+
while (sem == 0 && i < LOOP_TIMEOUT) {
482+
udelay(1);
483+
i += 1;
484+
}
539485

540-
if (iommu->reset_in_progress)
486+
if (i == LOOP_TIMEOUT) {
487+
pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
488+
iommu->reset_in_progress = true;
541489
reset_iommu_command_buffer(iommu);
490+
}
542491

543492
return 0;
544493
}

0 commit comments

Comments
 (0)