Skip to content

Commit a1abdf1

Browse files
benEngbenEng
authored
[BEAM-3713] Move PostCommit_Python from nosetest to pytest (apache#14859)
1 parent e4abe0d commit a1abdf1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+175
-187
lines changed

‎.test-infra/jenkins/job_PostCommit_Python.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ ALL_SUPPORTED_VERSIONS.each { pythonVersion ->
3333
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
3434

3535
publishers {
36-
archiveJunit('**/nosetests*.xml')
36+
archiveJunit('**/pytest*.xml')
3737
}
3838

3939
// Execute shell command to test Python SDK.

‎sdks/python/apache_beam/examples/complete/autocomplete_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
import unittest
2323

24-
from nose.plugins.attrib import attr
24+
import pytest
2525

2626
import apache_beam as beam
2727
from apache_beam.examples.complete import autocomplete
@@ -55,7 +55,7 @@ def test_top_prefixes(self):
5555
('that', ((1, 'that'), )),
5656
]))
5757

58-
@attr('IT')
58+
@pytest.mark.it_postcommit
5959
def test_autocomplete_it(self):
6060
with TestPipeline(is_integration_test=True) as p:
6161
words = p | beam.io.ReadFromText(self.KINGLEAR_INPUT)

‎sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
Code: beam/sdks/python/apache_beam/examples/complete/game/game_stats.py
2121
Usage:
2222
23-
python setup.py nosetests --test-pipeline-options=" \
23+
pytest --test-pipeline-options=" \
2424
--runner=TestDataflowRunner \
2525
--project=... \
2626
--region=... \
@@ -38,8 +38,8 @@
3838
import unittest
3939
import uuid
4040

41+
import pytest
4142
from hamcrest.core.core.allof import all_of
42-
from nose.plugins.attrib import attr
4343

4444
from apache_beam.examples.complete.game import game_stats
4545
from apache_beam.io.gcp.tests import utils
@@ -103,7 +103,7 @@ def _cleanup_pubsub(self):
103103
test_utils.cleanup_subscriptions(self.sub_client, [self.input_sub])
104104
test_utils.cleanup_topics(self.pub_client, [self.input_topic])
105105

106-
@attr('IT')
106+
@pytest.mark.it_postcommit
107107
def test_game_stats_it(self):
108108
state_verifier = PipelineStateMatcher(PipelineState.RUNNING)
109109

‎sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
Code: beam/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
2121
Usage:
2222
23-
python setup.py nosetests --test-pipeline-options=" \
23+
pytest --test-pipeline-options=" \
2424
--runner=TestDataflowRunner \
2525
--project=... \
2626
--region=... \
@@ -36,8 +36,8 @@
3636
import logging
3737
import unittest
3838

39+
import pytest
3940
from hamcrest.core.core.allof import all_of
40-
from nose.plugins.attrib import attr
4141

4242
from apache_beam.examples.complete.game import hourly_team_score
4343
from apache_beam.io.gcp.tests import utils
@@ -63,7 +63,7 @@ def setUp(self):
6363
self.dataset_ref = utils.create_bq_dataset(
6464
self.project, self.OUTPUT_DATASET)
6565

66-
@attr('IT')
66+
@pytest.mark.it_postcommit
6767
def test_hourly_team_score_it(self):
6868
state_verifier = PipelineStateMatcher(PipelineState.DONE)
6969
query = (

‎sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
Code: beam/sdks/python/apache_beam/examples/complete/game/leader_board.py
2121
Usage:
2222
23-
python setup.py nosetests --test-pipeline-options=" \
23+
pytest --test-pipeline-options=" \
2424
--runner=TestDataflowRunner \
2525
--project=... \
2626
--region=... \
@@ -38,8 +38,8 @@
3838
import unittest
3939
import uuid
4040

41+
import pytest
4142
from hamcrest.core.core.allof import all_of
42-
from nose.plugins.attrib import attr
4343

4444
from apache_beam.examples.complete.game import leader_board
4545
from apache_beam.io.gcp.tests import utils
@@ -104,7 +104,7 @@ def _cleanup_pubsub(self):
104104
test_utils.cleanup_subscriptions(self.sub_client, [self.input_sub])
105105
test_utils.cleanup_topics(self.pub_client, [self.input_topic])
106106

107-
@attr('IT')
107+
@pytest.mark.it_postcommit
108108
def test_leader_board_it(self):
109109
state_verifier = PipelineStateMatcher(PipelineState.RUNNING)
110110

‎sdks/python/apache_beam/examples/complete/game/user_score_it_test.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
Code: beam/sdks/python/apache_beam/examples/complete/game/user_score.py
2121
Usage:
2222
23-
python setup.py nosetests --test-pipeline-options=" \
23+
pytest --test-pipeline-options=" \
2424
--runner=TestDataflowRunner \
2525
--project=... \
2626
--region=... \
@@ -37,8 +37,8 @@
3737
import unittest
3838
import uuid
3939

40+
import pytest
4041
from hamcrest.core.core.allof import all_of
41-
from nose.plugins.attrib import attr
4242

4343
from apache_beam.examples.complete.game import user_score
4444
from apache_beam.runners.runner import PipelineState
@@ -60,7 +60,7 @@ def setUp(self):
6060
self.output = '/'.join(
6161
[self.test_pipeline.get_option('output'), self.uuid, 'results'])
6262

63-
@attr('IT')
63+
@pytest.mark.it_postcommit
6464
def test_user_score_it(self):
6565

6666
state_verifier = PipelineStateMatcher(PipelineState.DONE)

‎sdks/python/apache_beam/examples/complete/juliaset/juliaset/juliaset_test_it.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@
2424
import unittest
2525
import uuid
2626

27+
import pytest
2728
from hamcrest.core.core.allof import all_of
28-
from nose.plugins.attrib import attr
2929

3030
from apache_beam.examples.complete.juliaset.juliaset import juliaset
3131
from apache_beam.io.filesystems import FileSystems
@@ -34,7 +34,7 @@
3434
from apache_beam.testing.test_pipeline import TestPipeline
3535

3636

37-
@attr('IT')
37+
@pytest.mark.it_postcommit
3838
class JuliaSetTestIT(unittest.TestCase):
3939
GRID_SIZE = 1000
4040

‎sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
import time
2424
import unittest
2525

26+
import pytest
2627
from hamcrest.core.core.allof import all_of
27-
from nose.plugins.attrib import attr
2828

2929
from apache_beam.examples.cookbook import bigquery_tornadoes
3030
from apache_beam.io.gcp.tests import utils
@@ -42,7 +42,7 @@ class BigqueryTornadoesIT(unittest.TestCase):
4242
# from expected Bigquery table.
4343
DEFAULT_CHECKSUM = 'd860e636050c559a16a791aff40d6ad809d4daf0'
4444

45-
@attr('IT')
45+
@pytest.mark.it_postcommit
4646
def test_bigquery_tornadoes_it(self):
4747
test_pipeline = TestPipeline(is_integration_test=True)
4848

‎sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
import time
2424
import unittest
2525

26+
import pytest
2627
from hamcrest.core.core.allof import all_of
27-
from nose.plugins.attrib import attr
2828

2929
from apache_beam.testing.pipeline_verifiers import FileChecksumMatcher
3030
from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
@@ -42,7 +42,7 @@ class DatastoreWordCountIT(unittest.TestCase):
4242
DATASTORE_WORDCOUNT_KIND = "DatastoreWordCount"
4343
EXPECTED_CHECKSUM = '826f69ed0275858c2e098f1e8407d4e3ba5a4b3f'
4444

45-
@attr('IT')
45+
@pytest.mark.it_postcommit
4646
def test_datastore_wordcount_it(self):
4747
test_pipeline = TestPipeline(is_integration_test=True)
4848
kind = self.DATASTORE_WORDCOUNT_KIND

‎sdks/python/apache_beam/examples/dataframe/flight_delays_it_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
import uuid
2929

3030
import pandas as pd
31-
from nose.plugins.attrib import attr
31+
import pytest
3232

3333
from apache_beam.examples.dataframe import flight_delays
3434
from apache_beam.io.filesystems import FileSystems
@@ -100,7 +100,7 @@ def setUp(self):
100100
def tearDown(self):
101101
FileSystems.delete([self.outdir + '/'])
102102

103-
@attr('IT')
103+
@pytest.mark.it_postcommit
104104
def test_flight_delays(self):
105105
flight_delays.run_flight_delay_pipeline(
106106
self.test_pipeline,

‎sdks/python/apache_beam/examples/dataframe/taxiride_it_test.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import uuid
2626

2727
import pandas as pd
28-
from nose.plugins.attrib import attr
28+
import pytest
2929

3030
from apache_beam.examples.dataframe import taxiride
3131
from apache_beam.io.filesystems import FileSystems
@@ -44,7 +44,7 @@ def setUp(self):
4444
def tearDown(self):
4545
FileSystems.delete([self.outdir + '/'])
4646

47-
@attr('IT')
47+
@pytest.mark.it_postcommit
4848
def test_aggregation(self):
4949
taxiride.run_aggregation_pipeline(
5050
self.test_pipeline,
@@ -71,7 +71,7 @@ def read_csv(path):
7171

7272
pd.testing.assert_frame_equal(expected, result)
7373

74-
@attr('IT')
74+
@pytest.mark.it_postcommit
7575
def test_enrich(self):
7676
# Standard workers OOM with the enrich pipeline
7777
self.test_pipeline.get_pipeline_options().view_as(

‎sdks/python/apache_beam/examples/fastavro_it_test.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
Usage:
2525
2626
DataFlowRunner:
27-
python setup.py nosetests --tests apache_beam.examples.fastavro_it_test \
27+
pytest apache_beam/examples/fastavro_it_test.py \
2828
--test-pipeline-options="
2929
--runner=TestDataflowRunner
3030
--project=...
@@ -36,7 +36,7 @@
3636
"
3737
3838
DirectRunner:
39-
python setup.py nosetests --tests apache_beam.examples.fastavro_it_test \
39+
pytest apache_beam/examples/fastavro_it_test.py \
4040
--test-pipeline-options="
4141
--output=/tmp
4242
--records=5000
@@ -50,9 +50,9 @@
5050
import unittest
5151
import uuid
5252

53+
import pytest
5354
from avro.schema import Parse
5455
from fastavro import parse_schema
55-
from nose.plugins.attrib import attr
5656

5757
from apache_beam.io.avroio import ReadAllFromAvro
5858
from apache_beam.io.avroio import WriteToAvro
@@ -98,7 +98,7 @@ def setUp(self):
9898
self.uuid = str(uuid.uuid4())
9999
self.output = '/'.join([self.test_pipeline.get_option('output'), self.uuid])
100100

101-
@attr('IT')
101+
@pytest.mark.it_postcommit
102102
def test_avro_it(self):
103103
num_records = self.test_pipeline.get_option('records')
104104
num_records = int(num_records) if num_records else 1000000

‎sdks/python/apache_beam/examples/streaming_wordcount_debugging_it_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
import unittest
2424
import uuid
2525

26+
import pytest
2627
from hamcrest.core.core.allof import all_of
27-
from nose.plugins.attrib import attr
2828

2929
from apache_beam.examples import streaming_wordcount_debugging
3030
from apache_beam.io.gcp.tests.pubsub_matcher import PubSubMessageMatcher
@@ -94,7 +94,7 @@ def tearDown(self):
9494
test_utils.cleanup_topics(
9595
self.pub_client, [self.input_topic, self.output_topic])
9696

97-
@attr('IT')
97+
@pytest.mark.it_postcommit
9898
@unittest.skip(
9999
"Skipped due to [BEAM-3377]: assert_that not working for streaming")
100100
def test_streaming_wordcount_debugging_it(self):

‎sdks/python/apache_beam/examples/streaming_wordcount_it_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
import unittest
2424
import uuid
2525

26+
import pytest
2627
from hamcrest.core.core.allof import all_of
27-
from nose.plugins.attrib import attr
2828

2929
from apache_beam.examples import streaming_wordcount
3030
from apache_beam.io.gcp.tests.pubsub_matcher import PubSubMessageMatcher
@@ -77,7 +77,7 @@ def tearDown(self):
7777
test_utils.cleanup_topics(
7878
self.pub_client, [self.input_topic, self.output_topic])
7979

80-
@attr('IT')
80+
@pytest.mark.it_postcommit
8181
def test_streaming_wordcount_it(self):
8282
# Build expected dataset.
8383
expected_msg = [('%d: 1' % num).encode('utf-8')

‎sdks/python/apache_beam/examples/wordcount_it_test.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626

2727
import pytest
2828
from hamcrest.core.core.allof import all_of
29-
from nose.plugins.attrib import attr
3029

3130
from apache_beam.examples import wordcount
3231
from apache_beam.testing.load_tests.load_test_metrics_utils import InfluxDBMetricsPublisherOptions
@@ -39,19 +38,16 @@
3938

4039
class WordCountIT(unittest.TestCase):
4140

42-
# Enable nose tests running in parallel
43-
_multiprocess_can_split_ = True
44-
4541
# The default checksum is a SHA-1 hash generated from a sorted list of
4642
# lines read from expected output. This value corresponds to the default
4743
# input of WordCount example.
4844
DEFAULT_CHECKSUM = '33535a832b7db6d78389759577d4ff495980b9c0'
4945

50-
@attr('IT')
46+
@pytest.mark.it_postcommit
5147
def test_wordcount_it(self):
5248
self._run_wordcount_it(wordcount.run)
5349

54-
@attr('IT')
50+
@pytest.mark.it_postcommit
5551
@pytest.mark.it_validatescontainer
5652
def test_wordcount_fnapi_it(self):
5753
self._run_wordcount_it(wordcount.run, experiment='beam_fn_api')

‎sdks/python/apache_beam/io/fileio_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
import uuid
2929
import warnings
3030

31+
import pytest
3132
from hamcrest.library.text import stringmatches
32-
from nose.plugins.attrib import attr
3333

3434
import apache_beam as beam
3535
from apache_beam.io import fileio
@@ -291,7 +291,7 @@ class MatchIntegrationTest(unittest.TestCase):
291291
def setUp(self):
292292
self.test_pipeline = TestPipeline(is_integration_test=True)
293293

294-
@attr('IT')
294+
@pytest.mark.it_postcommit
295295
def test_transform_on_gcs(self):
296296
args = self.test_pipeline.get_full_options_as_args()
297297

0 commit comments

Comments
 (0)