Skip to content

Commit 3065267

Browse files
Matthew R. Ochsmartinkpetersen
Matthew R. Ochs
authored and
martinkpetersen
committed
scsi: cxlflash: Add hardware queues attribute
As staging for supporting multiple hardware queues, add an attribute to show and set the current number of hardware queues for the host. Support specifying a hard limit or a CPU affinitized value. This will allow the number of hardware queues to be tuned by a system administrator. Signed-off-by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com> Signed-off-by: Uma Krishnan <ukrishn@linux.vnet.ibm.com> Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
1 parent bfc0bab commit 3065267

File tree

2 files changed

+106
-16
lines changed

2 files changed

+106
-16
lines changed

‎drivers/scsi/cxlflash/common.h

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ extern const struct file_operations cxlflash_cxl_fops;
6060
/* SQ for master issued cmds */
6161
#define NUM_SQ_ENTRY CXLFLASH_MAX_CMDS
6262

63-
#define CXLFLASH_NUM_HWQS 1
63+
/* Hardware queue definitions */
64+
#define CXLFLASH_DEF_HWQS 1
65+
#define CXLFLASH_MAX_HWQS 8
6466
#define PRIMARY_HWQ 0
6567

6668

@@ -201,7 +203,7 @@ struct hwq {
201203
} __aligned(cache_line_size());
202204

203205
struct afu {
204-
struct hwq hwqs[CXLFLASH_NUM_HWQS];
206+
struct hwq hwqs[CXLFLASH_MAX_HWQS];
205207
int (*send_cmd)(struct afu *, struct afu_cmd *);
206208
void (*context_reset)(struct afu_cmd *);
207209

@@ -211,6 +213,8 @@ struct afu {
211213
atomic_t cmds_active; /* Number of currently active AFU commands */
212214
u64 hb;
213215
u32 internal_lun; /* User-desired LUN mode for this AFU */
216+
u32 num_hwqs; /* Number of hardware queues */
217+
u32 desired_hwqs; /* Desired h/w queues, effective on AFU reset */
214218

215219
char version[16];
216220
u64 interface_version;
@@ -221,7 +225,7 @@ struct afu {
221225

222226
static inline struct hwq *get_hwq(struct afu *afu, u32 index)
223227
{
224-
WARN_ON(index >= CXLFLASH_NUM_HWQS);
228+
WARN_ON(index >= CXLFLASH_MAX_HWQS);
225229

226230
return &afu->hwqs[index];
227231
}

‎drivers/scsi/cxlflash/main.c

Lines changed: 99 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,7 @@ static void stop_afu(struct cxlflash_cfg *cfg)
566566
ssleep(1);
567567

568568
if (afu_is_irqpoll_enabled(afu)) {
569-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
569+
for (i = 0; i < afu->num_hwqs; i++) {
570570
hwq = get_hwq(afu, i);
571571

572572
irq_poll_disable(&hwq->irqpoll);
@@ -676,13 +676,13 @@ static void term_afu(struct cxlflash_cfg *cfg)
676676
* 2) Unmap the problem state area
677677
* 3) Stop each master context
678678
*/
679-
for (k = CXLFLASH_NUM_HWQS - 1; k >= 0; k--)
679+
for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)
680680
term_intr(cfg, UNMAP_THREE, k);
681681

682682
if (cfg->afu)
683683
stop_afu(cfg);
684684

685-
for (k = CXLFLASH_NUM_HWQS - 1; k >= 0; k--)
685+
for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)
686686
term_mc(cfg, k);
687687

688688
dev_dbg(dev, "%s: returning\n", __func__);
@@ -823,6 +823,7 @@ static int alloc_mem(struct cxlflash_cfg *cfg)
823823
goto out;
824824
}
825825
cfg->afu->parent = cfg;
826+
cfg->afu->desired_hwqs = CXLFLASH_DEF_HWQS;
826827
cfg->afu->afu_map = NULL;
827828
out:
828829
return rc;
@@ -1116,7 +1117,7 @@ static void afu_err_intr_init(struct afu *afu)
11161117
/* IOARRIN yet), so there is nothing to clear. */
11171118

11181119
/* set LISN#, it is always sent to the context that wrote IOARRIN */
1119-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
1120+
for (i = 0; i < afu->num_hwqs; i++) {
11201121
hwq = get_hwq(afu, i);
11211122

11221123
writeq_be(SISL_MSI_SYNC_ERROR, &hwq->host_map->ctx_ctrl);
@@ -1551,7 +1552,7 @@ static void init_pcr(struct cxlflash_cfg *cfg)
15511552
}
15521553

15531554
/* Copy frequently used fields into hwq */
1554-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
1555+
for (i = 0; i < afu->num_hwqs; i++) {
15551556
hwq = get_hwq(afu, i);
15561557

15571558
hwq->ctx_hndl = (u16) cxl_process_element(hwq->ctx);
@@ -1586,7 +1587,7 @@ static int init_global(struct cxlflash_cfg *cfg)
15861587
}
15871588

15881589
/* Set up RRQ and SQ in HWQ for master issued cmds */
1589-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
1590+
for (i = 0; i < afu->num_hwqs; i++) {
15901591
hwq = get_hwq(afu, i);
15911592
hmap = hwq->host_map;
15921593

@@ -1640,7 +1641,7 @@ static int init_global(struct cxlflash_cfg *cfg)
16401641
/* Set up master's own CTX_CAP to allow real mode, host translation */
16411642
/* tables, afu cmds and read/write GSCSI cmds. */
16421643
/* First, unlock ctx_cap write by reading mbox */
1643-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
1644+
for (i = 0; i < afu->num_hwqs; i++) {
16441645
hwq = get_hwq(afu, i);
16451646

16461647
(void)readq_be(&hwq->ctrl_map->mbox_r); /* unlock ctx_cap */
@@ -1670,7 +1671,7 @@ static int start_afu(struct cxlflash_cfg *cfg)
16701671
init_pcr(cfg);
16711672

16721673
/* Initialize each HWQ */
1673-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
1674+
for (i = 0; i < afu->num_hwqs; i++) {
16741675
hwq = get_hwq(afu, i);
16751676

16761677
/* After an AFU reset, RRQ entries are stale, clear them */
@@ -1888,7 +1889,8 @@ static int init_afu(struct cxlflash_cfg *cfg)
18881889

18891890
cxl_perst_reloads_same_image(cfg->cxl_afu, true);
18901891

1891-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
1892+
afu->num_hwqs = afu->desired_hwqs;
1893+
for (i = 0; i < afu->num_hwqs; i++) {
18921894
rc = init_mc(cfg, i);
18931895
if (rc) {
18941896
dev_err(dev, "%s: init_mc failed rc=%d index=%d\n",
@@ -1939,7 +1941,7 @@ static int init_afu(struct cxlflash_cfg *cfg)
19391941
}
19401942

19411943
afu_err_intr_init(cfg->afu);
1942-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
1944+
for (i = 0; i < afu->num_hwqs; i++) {
19431945
hwq = get_hwq(afu, i);
19441946

19451947
spin_lock_init(&hwq->rrin_slock);
@@ -1953,7 +1955,7 @@ static int init_afu(struct cxlflash_cfg *cfg)
19531955
return rc;
19541956

19551957
err1:
1956-
for (i = CXLFLASH_NUM_HWQS - 1; i >= 0; i--) {
1958+
for (i = afu->num_hwqs - 1; i >= 0; i--) {
19571959
term_intr(cfg, UNMAP_THREE, i);
19581960
term_mc(cfg, i);
19591961
}
@@ -2550,7 +2552,7 @@ static ssize_t irqpoll_weight_store(struct device *dev,
25502552
}
25512553

25522554
if (afu_is_irqpoll_enabled(afu)) {
2553-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
2555+
for (i = 0; i < afu->num_hwqs; i++) {
25542556
hwq = get_hwq(afu, i);
25552557

25562558
irq_poll_disable(&hwq->irqpoll);
@@ -2560,7 +2562,7 @@ static ssize_t irqpoll_weight_store(struct device *dev,
25602562
afu->irqpoll_weight = weight;
25612563

25622564
if (weight > 0) {
2563-
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
2565+
for (i = 0; i < afu->num_hwqs; i++) {
25642566
hwq = get_hwq(afu, i);
25652567

25662568
irq_poll_init(&hwq->irqpoll, weight, cxlflash_irqpoll);
@@ -2570,6 +2572,88 @@ static ssize_t irqpoll_weight_store(struct device *dev,
25702572
return count;
25712573
}
25722574

2575+
/**
2576+
* num_hwqs_show() - presents the number of hardware queues for the host
2577+
* @dev: Generic device associated with the host.
2578+
* @attr: Device attribute representing the number of hardware queues.
2579+
* @buf: Buffer of length PAGE_SIZE to report back the number of hardware
2580+
* queues in ASCII.
2581+
*
2582+
* Return: The size of the ASCII string returned in @buf.
2583+
*/
2584+
static ssize_t num_hwqs_show(struct device *dev,
2585+
struct device_attribute *attr, char *buf)
2586+
{
2587+
struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev));
2588+
struct afu *afu = cfg->afu;
2589+
2590+
return scnprintf(buf, PAGE_SIZE, "%u\n", afu->num_hwqs);
2591+
}
2592+
2593+
/**
2594+
* num_hwqs_store() - sets the number of hardware queues for the host
2595+
* @dev: Generic device associated with the host.
2596+
* @attr: Device attribute representing the number of hardware queues.
2597+
* @buf: Buffer of length PAGE_SIZE containing the number of hardware
2598+
* queues in ASCII.
2599+
* @count: Length of data resizing in @buf.
2600+
*
2601+
* n > 0: num_hwqs = n
2602+
* n = 0: num_hwqs = num_online_cpus()
2603+
* n < 0: num_online_cpus() / abs(n)
2604+
*
2605+
* Return: The size of the ASCII string returned in @buf.
2606+
*/
2607+
static ssize_t num_hwqs_store(struct device *dev,
2608+
struct device_attribute *attr,
2609+
const char *buf, size_t count)
2610+
{
2611+
struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev));
2612+
struct afu *afu = cfg->afu;
2613+
int rc;
2614+
int nhwqs, num_hwqs;
2615+
2616+
rc = kstrtoint(buf, 10, &nhwqs);
2617+
if (rc)
2618+
return -EINVAL;
2619+
2620+
if (nhwqs >= 1)
2621+
num_hwqs = nhwqs;
2622+
else if (nhwqs == 0)
2623+
num_hwqs = num_online_cpus();
2624+
else
2625+
num_hwqs = num_online_cpus() / abs(nhwqs);
2626+
2627+
afu->desired_hwqs = min(num_hwqs, CXLFLASH_MAX_HWQS);
2628+
WARN_ON_ONCE(afu->desired_hwqs == 0);
2629+
2630+
retry:
2631+
switch (cfg->state) {
2632+
case STATE_NORMAL:
2633+
cfg->state = STATE_RESET;
2634+
drain_ioctls(cfg);
2635+
cxlflash_mark_contexts_error(cfg);
2636+
rc = afu_reset(cfg);
2637+
if (rc)
2638+
cfg->state = STATE_FAILTERM;
2639+
else
2640+
cfg->state = STATE_NORMAL;
2641+
wake_up_all(&cfg->reset_waitq);
2642+
break;
2643+
case STATE_RESET:
2644+
wait_event(cfg->reset_waitq, cfg->state != STATE_RESET);
2645+
if (cfg->state == STATE_NORMAL)
2646+
goto retry;
2647+
default:
2648+
/* Ideally should not happen */
2649+
dev_err(dev, "%s: Device is not ready, state=%d\n",
2650+
__func__, cfg->state);
2651+
break;
2652+
}
2653+
2654+
return count;
2655+
}
2656+
25732657
/**
25742658
* mode_show() - presents the current mode of the device
25752659
* @dev: Generic device associated with the device.
@@ -2601,6 +2685,7 @@ static DEVICE_ATTR_RO(port1_lun_table);
26012685
static DEVICE_ATTR_RO(port2_lun_table);
26022686
static DEVICE_ATTR_RO(port3_lun_table);
26032687
static DEVICE_ATTR_RW(irqpoll_weight);
2688+
static DEVICE_ATTR_RW(num_hwqs);
26042689

26052690
static struct device_attribute *cxlflash_host_attrs[] = {
26062691
&dev_attr_port0,
@@ -2614,6 +2699,7 @@ static struct device_attribute *cxlflash_host_attrs[] = {
26142699
&dev_attr_port2_lun_table,
26152700
&dev_attr_port3_lun_table,
26162701
&dev_attr_irqpoll_weight,
2702+
&dev_attr_num_hwqs,
26172703
NULL
26182704
};
26192705

0 commit comments

Comments
 (0)