From 8ca8a43b3b8369c905af3dbd16c38c062907db1d Mon Sep 17 00:00:00 2001 From: bradmiro Date: Tue, 12 Nov 2019 14:48:03 -0500 Subject: [PATCH 1/5] adding sample for cluster create --- dataproc/create_cluster.py | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 dataproc/create_cluster.py diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py new file mode 100644 index 00000000000..d756005d9da --- /dev/null +++ b/dataproc/create_cluster.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +def create_cluster(project_id, region, cluster_name): + # [START create_cluster] + from google.cloud import dataproc_v1 + + # TODO(developer): Uncomment and set the following variables + # project_id = 'YOUR_PROJECT_ID' + # region = 'YOUR_CLUSTER_REGION' + # cluster_name = 'YOUR_CLUSTER_NAME' + + # Create client + client = dataproc_v1.ClusterControllerClient({ + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) + }) + + # Create cluster config + cluster = { + 'project_id': project, + 'cluster_name': cluster_name, + 'config': { + 'master_config': { + 'num_instances': 1, + 'machine_type_uri': 'n1-standard-1' + }, + 'worker_config': { + 'num_instances': 2, + 'machine_type_uri': 'n1-standard-1' + } + } + } + + # Define a callback + def callback(operation_future): + result = operation_future.result() + print(result) + + # Submit cluster creation request + response = client.create_cluster(project_id, region, cluster) + + # Add callback to cluster creation request + response.add_done_callback(callback) + + print(response.metadata()) + # [END create_cluster] + + From d8e174f3e5c0b67cc946d87ad3ad7565934e47f3 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Tue, 12 Nov 2019 14:49:14 -0500 Subject: [PATCH 2/5] small fix --- dataproc/create_cluster.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py index d756005d9da..96e4bde2e49 100644 --- a/dataproc/create_cluster.py +++ b/dataproc/create_cluster.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - def create_cluster(project_id, region, cluster_name): # [START create_cluster] from google.cloud import dataproc_v1 From 6721673c291423fc0af229ad993d160de593764a Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 15 Nov 2019 16:58:25 -0500 Subject: [PATCH 3/5] Add create cluster samples --- dataproc/create_cluster.py | 29 +++++++++------------- dataproc/create_cluster_test.py | 44 +++++++++++++++++++++++++++++++++ dataproc/requirements.txt | 2 +- 3 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 dataproc/create_cluster_test.py diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py index 96e4bde2e49..112656320ed 100644 --- a/dataproc/create_cluster.py +++ b/dataproc/create_cluster.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def create_cluster(project_id, region, cluster_name): # [START create_cluster] from google.cloud import dataproc_v1 @@ -23,14 +24,14 @@ def create_cluster(project_id, region, cluster_name): # region = 'YOUR_CLUSTER_REGION' # cluster_name = 'YOUR_CLUSTER_NAME' - # Create client - client = dataproc_v1.ClusterControllerClient({ + # Create a client with the endpoint set to the desired cluster region + client = dataproc_v1.ClusterControllerClient(client_options={ 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) }) - # Create cluster config + # Create the cluster config cluster = { - 'project_id': project, + 'project_id': project_id, 'cluster_name': cluster_name, 'config': { 'master_config': { @@ -44,18 +45,10 @@ def create_cluster(project_id, region, cluster_name): } } - # Define a callback - def callback(operation_future): - result = operation_future.result() - print(result) - - # Submit cluster creation request - response = client.create_cluster(project_id, region, cluster) - - # Add callback to cluster creation request - response.add_done_callback(callback) - - print(response.metadata()) - # [END create_cluster] - + # Create the cluster + operation = client.create_cluster(project_id, region, cluster) + result = operation.result() + # Output a success message + print('Cluster created successfully: {}'.format(result.cluster_name)) + # [END create_cluster] diff --git a/dataproc/create_cluster_test.py b/dataproc/create_cluster_test.py new file mode 100644 index 00000000000..7e8f882df65 --- /dev/null +++ b/dataproc/create_cluster_test.py @@ -0,0 +1,44 @@ +# Copyright 2019 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid +import pytest + +from google.cloud import dataproc + +import create_cluster + +PROJECT_ID = os.environ['GCLOUD_PROJECT'] +REGION = 'us-central1' +CLUSTER_NAME = 'test-cluster-{}'.format(str(uuid.uuid4())) + + +@pytest.fixture(autouse=True) +def teardown(): + yield + + client = dataproc.ClusterControllerClient(client_options={ + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(REGION) + }) + # Client library function + client.delete_cluster(PROJECT_ID, REGION, CLUSTER_NAME) + + +def test_cluster_create(capsys): + # Wrapper function for client library function + create_cluster.create_cluster(PROJECT_ID, REGION, CLUSTER_NAME) + + out, _ = capsys.readouterr() + assert CLUSTER_NAME in out diff --git a/dataproc/requirements.txt b/dataproc/requirements.txt index 81a0a72bf2b..0ffe752a6ea 100644 --- a/dataproc/requirements.txt +++ b/dataproc/requirements.txt @@ -3,4 +3,4 @@ google-auth==1.6.3 google-auth-httplib2==0.0.3 google-cloud==0.34.0 google-cloud-storage==1.19.1 -google-cloud-dataproc==0.5.0 +google-cloud-dataproc==0.6.1 From 603b638d2d576273e00fd64450f1217350e45d08 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 15 Nov 2019 18:20:46 -0500 Subject: [PATCH 4/5] Fixed copyright, added 'dataproc' to region tag and changed imports from 'dataproc' to 'dataproc_v1' --- dataproc/create_cluster.py | 8 ++++---- dataproc/create_cluster_test.py | 4 ++-- .../{dataproc_e2e_test.py => dataproc_e2e_donttest.py} | 0 3 files changed, 6 insertions(+), 6 deletions(-) rename dataproc/{dataproc_e2e_test.py => dataproc_e2e_donttest.py} (100%) diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py index 112656320ed..9c4748116c8 100644 --- a/dataproc/create_cluster.py +++ b/dataproc/create_cluster.py @@ -16,8 +16,8 @@ def create_cluster(project_id, region, cluster_name): - # [START create_cluster] - from google.cloud import dataproc_v1 + # [START dataproc_create_cluster] + from google.cloud import dataproc_v1 as dataproc # TODO(developer): Uncomment and set the following variables # project_id = 'YOUR_PROJECT_ID' @@ -25,7 +25,7 @@ def create_cluster(project_id, region, cluster_name): # cluster_name = 'YOUR_CLUSTER_NAME' # Create a client with the endpoint set to the desired cluster region - client = dataproc_v1.ClusterControllerClient(client_options={ + client = dataproc.ClusterControllerClient(client_options={ 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) }) @@ -51,4 +51,4 @@ def create_cluster(project_id, region, cluster_name): # Output a success message print('Cluster created successfully: {}'.format(result.cluster_name)) - # [END create_cluster] + # [END dataproc_create_cluster] diff --git a/dataproc/create_cluster_test.py b/dataproc/create_cluster_test.py index 7e8f882df65..d58a1d0b655 100644 --- a/dataproc/create_cluster_test.py +++ b/dataproc/create_cluster_test.py @@ -1,4 +1,4 @@ -# Copyright 2019 Google, LLC. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ import uuid import pytest -from google.cloud import dataproc +from google.cloud import dataproc_v1 as dataproc import create_cluster diff --git a/dataproc/dataproc_e2e_test.py b/dataproc/dataproc_e2e_donttest.py similarity index 100% rename from dataproc/dataproc_e2e_test.py rename to dataproc/dataproc_e2e_donttest.py From 44207b9b92fde9fe18a13c14485c1f937aba7e35 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 15 Nov 2019 18:22:00 -0500 Subject: [PATCH 5/5] Fix copyright in create_cluster.py --- dataproc/create_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py index 9c4748116c8..d893a142fa1 100644 --- a/dataproc/create_cluster.py +++ b/dataproc/create_cluster.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2019 Google, LLC. +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.