Skip to content

Commit

Permalink
fix: data skew when replace into reshuffle segments (#12479)
Browse files Browse the repository at this point in the history
* fix: data skew when replace into reshuffle segments

* fix
  • Loading branch information
SkyFan2002 authored Aug 16, 2023
1 parent 0461806 commit e33eec4
Showing 1 changed file with 10 additions and 0 deletions.
10 changes: 10 additions & 0 deletions src/query/service/src/schedulers/fragments/plan_fragment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ impl PlanFragment {
Ok(fragment_actions)
}

// TODO(sky): reuse Partitions reshuffle
fn reshuffle<T: Clone>(
executors: Vec<String>,
partitions: Vec<T>,
Expand All @@ -219,6 +220,15 @@ impl PlanFragment {
let num_executors = executors.len();
let mut executors_sorted = executors;
executors_sorted.sort();

let mut parts = partitions
.into_iter()
.enumerate()
.map(|(idx, p)| (idx % num_executors, p))
.collect::<Vec<_>>();
parts.sort_by(|a, b| a.0.cmp(&b.0));
let partitions: Vec<_> = parts.into_iter().map(|x| x.1).collect();

let mut executor_part = HashMap::default();
// the first num_parts % num_executors get parts_per_node parts
// the remaining get parts_per_node - 1 parts
Expand Down

1 comment on commit e33eec4

@vercel
Copy link

@vercel vercel bot commented on e33eec4 Aug 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

databend – ./

databend-git-main-databend.vercel.app
databend.rs
databend-databend.vercel.app
databend.vercel.app

Please sign in to comment.