Skip to content

feat: Allow df.drop to take an index object #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 22 commits into from
Oct 4, 2023
Merged

feat: Allow df.drop to take an index object #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
93d3a00
feat: Allow passing index objects to df.drop
Sep 27, 2023
ee8eb0d
remove notebook files
Sep 27, 2023
0fc8dd7
add first implementation for df.drop(index)
Sep 28, 2023
5cdfff9
use index_columns property
Sep 28, 2023
99f2f59
don't use _expr.keys()
Sep 28, 2023
f4db207
fix order bug and add test
Sep 28, 2023
ec7c6a7
fix index names
Sep 28, 2023
fd22b01
support multiindex
Sep 28, 2023
b0b0533
remove accidentally added files
Sep 28, 2023
829dd4f
Merge branch 'main' into b302296209-drop-rows-using-index
milkshakeiii Sep 28, 2023
9a5afdd
add type hint
Sep 28, 2023
794058b
Merge branch 'main' of github.com:googleapis/python-bigquery-datafram…
Sep 28, 2023
5c659bf
Merge branch 'b302296209-drop-rows-using-index' of github.com:googlea…
Sep 28, 2023
451b901
remove debug print statements
Sep 28, 2023
3ffa42d
Merge branch 'main' of github.com:googleapis/python-bigquery-datafram…
Oct 2, 2023
6a5ac4f
fix: fix df/series.iloc by list with multiindex
Oct 2, 2023
9e7b667
Merge branch 'main' of github.com:googleapis/python-bigquery-datafram…
Oct 3, 2023
01334bd
complete merge
Oct 3, 2023
a400b1d
remove unneeded isinstance
Oct 3, 2023
afbf8c3
refactor _drop_by_index
Oct 3, 2023
aaa6b58
Merge branch 'main' into b302296209-drop-rows-using-index
tswast Oct 4, 2023
7524adb
Merge branch 'main' into b302296209-drop-rows-using-index
milkshakeiii Oct 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions bigframes/dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -937,7 +937,7 @@ def drop(
columns = labels

block = self._block
if index:
if index is not None:
level_id = self._resolve_levels(level or 0)[0]

if utils.is_list_like(index):
Expand All @@ -947,6 +947,8 @@ def drop(
block, condition_id = block.apply_unary_op(
inverse_condition_id, ops.invert_op
)
elif isinstance(index, indexes.Index):
return self._drop_by_index(index)
else:
block, condition_id = block.apply_unary_op(
level_id, ops.partial_right(ops.ne_op, index)
Expand All @@ -956,10 +958,31 @@ def drop(
)
if columns:
block = block.drop_columns(self._sql_names(columns))
if not index and not columns:
if index is None and not columns:
raise ValueError("Must specify 'labels' or 'index'/'columns")
return DataFrame(block)

def _drop_by_index(self, index: indexes.Index) -> DataFrame:
block = index._data._get_block()
block, ordering_col = block.promote_offsets()
joined_index, (get_column_left, get_column_right) = self._block.index.join(
block.index
)

new_ordering_col = get_column_right(ordering_col)
drop_block = joined_index._block
drop_block, drop_col = drop_block.apply_unary_op(
new_ordering_col,
ops.isnull_op,
)

drop_block = drop_block.filter(drop_col)
original_columns = [
get_column_left(column) for column in self._block.value_columns
]
drop_block = drop_block.select_columns(original_columns)
return DataFrame(drop_block)

def droplevel(self, level: LevelsType, axis: int | str = 0):
axis_n = utils.get_axis_number(axis)
if axis_n == 0:
Expand Down
55 changes: 55 additions & 0 deletions tests/system/small/test_dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,61 @@ def test_drop_index(scalars_dfs):
pd.testing.assert_frame_equal(pd_result, bf_result)


def test_drop_pandas_index(scalars_dfs):
scalars_df, scalars_pandas_df = scalars_dfs
drop_index = scalars_pandas_df.iloc[[4, 1, 2]].index

pd_result = scalars_pandas_df.drop(index=drop_index)
bf_result = scalars_df.drop(index=drop_index).to_pandas()

pd.testing.assert_frame_equal(pd_result, bf_result)


def test_drop_bigframes_index(scalars_dfs):
scalars_df, scalars_pandas_df = scalars_dfs
drop_index = scalars_df.loc[[4, 1, 2]].index
drop_pandas_index = scalars_pandas_df.loc[[4, 1, 2]].index

pd_result = scalars_pandas_df.drop(index=drop_pandas_index)
bf_result = scalars_df.drop(index=drop_index).to_pandas()

pd.testing.assert_frame_equal(pd_result, bf_result)


def test_drop_bigframes_index_with_na(scalars_dfs):
scalars_df, scalars_pandas_df = scalars_dfs
scalars_df = scalars_df.copy()
scalars_pandas_df = scalars_pandas_df.copy()
scalars_df = scalars_df.set_index("bytes_col")
scalars_pandas_df = scalars_pandas_df.set_index("bytes_col")
drop_index = scalars_df.iloc[[3, 5]].index
drop_pandas_index = scalars_pandas_df.iloc[[3, 5]].index

pd_result = scalars_pandas_df.drop(index=drop_pandas_index) # drop_pandas_index)
bf_result = scalars_df.drop(index=drop_index).to_pandas()

pd.testing.assert_frame_equal(pd_result, bf_result)


def test_drop_bigframes_multiindex(scalars_dfs):
scalars_df, scalars_pandas_df = scalars_dfs
scalars_df = scalars_df.copy()
scalars_pandas_df = scalars_pandas_df.copy()
sub_df = scalars_df.iloc[[4, 1, 2]]
sub_pandas_df = scalars_pandas_df.iloc[[4, 1, 2]]
sub_df = sub_df.set_index(["bytes_col", "numeric_col"])
sub_pandas_df = sub_pandas_df.set_index(["bytes_col", "numeric_col"])
drop_index = sub_df.index
drop_pandas_index = sub_pandas_df.index

scalars_df = scalars_df.set_index(["bytes_col", "numeric_col"])
scalars_pandas_df = scalars_pandas_df.set_index(["bytes_col", "numeric_col"])
bf_result = scalars_df.drop(index=drop_index).to_pandas()
pd_result = scalars_pandas_df.drop(index=drop_pandas_index)

pd.testing.assert_frame_equal(pd_result, bf_result)


def test_drop_labels_axis_0(scalars_dfs):
scalars_df, scalars_pandas_df = scalars_dfs

Expand Down