Skip to content

Commit 421ca79

Browse files
codelipenghuicodelipenghui
authored
Fix the Kafka driver for the deployment for i3en.6xlarge (openmessaging#217)
* Fix the Kafka driver for the deployment for i3en.6xlarge - Fix ansible playbook - Set reset to true to for Kafka driver to make sure cleanup the existing topics - Add new configuration for Kafka high throughput and low latency * Add 2000k rate * Add 2000k rate * Add 2000k rate * Add max throughput for 4 producers and 1 consumer
1 parent b7b0ee1 commit 421ca79

20 files changed

+175
-6
lines changed

‎driver-kafka/deploy/ssd-deployment/deploy.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
fstype: xfs
3030
dev: '{{ item }}'
3131
with_items:
32-
- '/dev/nvme0n1'
3332
- '/dev/nvme1n1'
33+
- '/dev/nvme2n1'
3434
- name: Mount disks
3535
mount:
3636
path: "{{ item.path }}"
@@ -39,8 +39,8 @@
3939
opts: defaults,noatime,nodiscard
4040
state: mounted
4141
with_items:
42-
- { path: "/mnt/data-1", src: "/dev/nvme0n1" }
43-
- { path: "/mnt/data-2", src: "/dev/nvme1n1" }
42+
- { path: "/mnt/data-1", src: "/dev/nvme1n1" }
43+
- { path: "/mnt/data-2", src: "/dev/nvme2n1" }
4444
- name: Increase hard file descriptor limit
4545
pam_limits:
4646
domain: '*'

‎driver-kafka/deploy/ssd-deployment/provision-kafka-aws.tf

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,6 @@ resource "aws_instance" "kafka" {
127127
subnet_id = "${aws_subnet.benchmark_subnet.id}"
128128
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
129129
count = "${var.num_instances["kafka"]}"
130-
wait_for_fulfillment = true
131130

132131
tags = {
133132
Name = "kafka-${count.index}"
@@ -141,7 +140,6 @@ resource "aws_instance" "client" {
141140
subnet_id = "${aws_subnet.benchmark_subnet.id}"
142141
vpc_security_group_ids = ["${aws_security_group.benchmark_security_group.id}"]
143142
count = "${var.num_instances["client"]}"
144-
wait_for_fulfillment = true
145143

146144
tags = {
147145
Name = "kafka-client-${count.index}"

‎driver-kafka/kafka-big-batches-gzip.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-big-batches-lz4.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-big-batches-snappy.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-big-batches-zstd.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-big-batches.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-compression-gzip.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-compression-lz4.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-compression-snappy.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-compression-zstd.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-exactly-once.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-latency.yaml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
21+
name: Kafka
22+
driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
23+
24+
# Kafka client-specific configuration
25+
replicationFactor: 3
26+
reset: true
27+
28+
topicConfig: |
29+
min.insync.replicas=2
30+
31+
commonConfig: |
32+
bootstrap.servers=localhost:9092
33+
34+
producerConfig: |
35+
acks=all
36+
linger.ms=1
37+
batch.size=131072
38+
39+
consumerConfig: |
40+
auto.offset.reset=earliest
41+
enable.auto.commit=false
42+
max.partition.fetch.bytes=10485760

‎driver-kafka/kafka-no-linger.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2

‎driver-kafka/kafka-sync.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2424

2525
# Kafka client-specific configuration
2626
replicationFactor: 3
27+
reset: true
2728

2829
topicConfig: |
2930
min.insync.replicas=2

‎driver-kafka/kafka.yaml renamed to ‎driver-kafka/kafka-throughput.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ driverClass: io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver
2323

2424
# Kafka client-specific configuration
2525
replicationFactor: 3
26+
reset: true
2627

2728
topicConfig: |
2829
min.insync.replicas=2
@@ -32,7 +33,7 @@ commonConfig: |
3233
3334
producerConfig: |
3435
acks=all
35-
linger.ms=100
36+
linger.ms=1
3637
batch.size=1048576
3738
3839
consumerConfig: |

‎driver-pulsar/pulsar.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ client:
4545
producer:
4646
batchingEnabled: true
4747
batchingMaxPublishDelayMs: 1
48+
batchingMaxBytes: 131072
4849
blockIfQueueFull: true
4950
pendingQueueSize: 0
5051

‎workloads/1-topic-100-partitions-1kb-4p-4c-2000k.yaml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: 2000k rate 4 producers and 4 consumers on 1 topic / 100 partition
21+
22+
topics: 1
23+
partitionsPerTopic: 100
24+
25+
messageSize: 1024
26+
useRandomizedPayloads: true
27+
randomBytesRatio: 0.5
28+
randomizedPayloadPoolSize: 1000
29+
30+
subscriptionsPerTopic: 0
31+
consumerPerSubscription: 0
32+
producersPerTopic: 4
33+
34+
# Discover max-sustainable rate
35+
producerRate: 2000000
36+
37+
consumerBacklogSizeGB: 0
38+
testDurationMinutes: 5

‎workloads/1-topic-10000-partitions-1kb-4p-4c-2000k.yaml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: 2000k rate 4 producers and 4 consumers on 1 topic / 10000 partition
21+
22+
topics: 1
23+
partitionsPerTopic: 10000
24+
25+
messageSize: 1024
26+
useRandomizedPayloads: true
27+
randomBytesRatio: 0.5
28+
randomizedPayloadPoolSize: 1000
29+
30+
subscriptionsPerTopic: 0
31+
consumerPerSubscription: 0
32+
producersPerTopic: 4
33+
34+
# Discover max-sustainable rate
35+
producerRate: 2000000
36+
37+
consumerBacklogSizeGB: 0
38+
testDurationMinutes: 5

‎workloads/max-rate-1-topic-1-partition-4p-1c-1kb.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: max-rate-1-topic-1-partition-4p-1c-1kb
21+
22+
topics: 1
23+
partitionsPerTopic: 1
24+
25+
messageSize: 1024
26+
#payloadFile: "payload/payload-1Kb.data"
27+
useRandomizedPayloads: true
28+
randomBytesRatio: 0.5
29+
randomizedPayloadPoolSize: 1000
30+
31+
subscriptionsPerTopic: 1
32+
consumerPerSubscription: 1
33+
producersPerTopic: 4
34+
35+
# Discover max-sustainable rate
36+
producerRate: 10000000
37+
38+
consumerBacklogSizeGB: 0
39+
testDurationMinutes: 5

0 commit comments

Comments
 (0)