perf: avoid transmitting parity shards when the object is good #322
Conversation
|
|
|
You can try using use futures::stream::{FuturesUnordered, StreamExt};
let mut futures = Vec::with_capacity(num_readers);
for (i, reader) in self.readers.iter_mut().enumerate() {
if let Some(r) = reader {
futures.push(async move {
let mut buf = vec![0u8; shard_size];
match r.read(&mut buf).await {
Ok(n) => {
buf.truncate(n);
(i, Ok(buf))
}
Err(e) => (i, Err(Error::from(e))),
}
});
}
}
if futures.len() > self.data_shards {
let mut sets = FuturesUnordered::new();
let mut fns_iter = futures.into_iter();
for _ in 0..self.data_shards {
if let Some(f) = fns_iter.next() {
sets.push(f)
}
}
while let Some((i, result)) = sets.next().await {
match result {
Ok(v) => {}
Err(e) => {
// error!("Error reading shard {}: {}", i, e);
if let Some(f) = fns_iter.next() {
sets.push(f)
}
}
}
}
} |
|
@shiroleeee thanks for your advice, I'll improve this. |
cc37fa5 to
5048dc2
Compare
|
You've done great work! However, I think the logic for determining whether sufficient data shards are obtained could be further simplified. For example, as shown below, we can simply verify if the number of successful returns matches the total count of data shards. let mut success = 0;
while let Some((i, result)) = sets.next().await {
match result {
Ok(v) => {
shards[i] = Some(v);
success += 1;
}
Err(e) => {
errs[i] = Some(e);
if let Some(future) = fut_iter.next() {
sets.push(future);
}
continue;
}
}
if success == self.data_shards {
break;
}
} |
|
@shiroleeee You're right. Your implementation is actually equivalent to mine while being simpler. I've applied the changes in the new commit. |
Type of Change
Related Issues
#46
#50
Summary of Changes
When executing
GetObject, transmitting the data shards only when the object is fine. Read the parity shards only when some of the data shards are broken.I've added some unit tests and tested this implementation on a three-nodes RustFS setup, the object is available when one of the nodes is down or one part file is lost
Checklist
cargo fmt --allcargo clippy --all-targets --all-features -- -D warningscargo check --all-targetsImpact
Additional Notes
Thank you for your contribution! Please ensure your PR follows the community standards (CODE_OF_CONDUCT.md) and sign the CLA if this is your first contribution.