From f8565c81925ce8ab4261bdf0c12846c7d63998b6 Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Wed, 16 Mar 2016 09:17:05 -0700 Subject: [PATCH] Add Monitoring v3 Samples Refactor v2 samples in to separate directory Add v3 Samples Renamed auth to list_env --- monitoring/api/{ => v2}/README.md | 0 monitoring/api/{ => v2}/auth.py | 0 monitoring/api/{ => v2}/auth_test.py | 0 .../api/{ => v2}/labeled_custom_metric.py | 0 .../{ => v2}/labeled_custom_metric_test.py | 0 .../api/{ => v2}/lightweight_custom_metric.py | 0 .../lightweight_custom_metric_test.py | 0 monitoring/api/{ => v2}/requirements.txt | 0 monitoring/api/v3/README.md | 47 ++++ monitoring/api/v3/custom_metric.py | 202 ++++++++++++++++++ monitoring/api/v3/custom_metric_test.py | 82 +++++++ monitoring/api/v3/list_resources.py | 127 +++++++++++ monitoring/api/v3/list_resources_test.py | 60 ++++++ monitoring/api/v3/requirements.txt | 10 + 14 files changed, 528 insertions(+) rename monitoring/api/{ => v2}/README.md (100%) rename monitoring/api/{ => v2}/auth.py (100%) rename monitoring/api/{ => v2}/auth_test.py (100%) rename monitoring/api/{ => v2}/labeled_custom_metric.py (100%) rename monitoring/api/{ => v2}/labeled_custom_metric_test.py (100%) rename monitoring/api/{ => v2}/lightweight_custom_metric.py (100%) rename monitoring/api/{ => v2}/lightweight_custom_metric_test.py (100%) rename monitoring/api/{ => v2}/requirements.txt (100%) create mode 100644 monitoring/api/v3/README.md create mode 100644 monitoring/api/v3/custom_metric.py create mode 100644 monitoring/api/v3/custom_metric_test.py create mode 100644 monitoring/api/v3/list_resources.py create mode 100644 monitoring/api/v3/list_resources_test.py create mode 100644 monitoring/api/v3/requirements.txt diff --git a/monitoring/api/README.md b/monitoring/api/v2/README.md similarity index 100% rename from monitoring/api/README.md rename to monitoring/api/v2/README.md diff --git a/monitoring/api/auth.py b/monitoring/api/v2/auth.py similarity index 100% rename from monitoring/api/auth.py rename to monitoring/api/v2/auth.py diff --git a/monitoring/api/auth_test.py b/monitoring/api/v2/auth_test.py similarity index 100% rename from monitoring/api/auth_test.py rename to monitoring/api/v2/auth_test.py diff --git a/monitoring/api/labeled_custom_metric.py b/monitoring/api/v2/labeled_custom_metric.py similarity index 100% rename from monitoring/api/labeled_custom_metric.py rename to monitoring/api/v2/labeled_custom_metric.py diff --git a/monitoring/api/labeled_custom_metric_test.py b/monitoring/api/v2/labeled_custom_metric_test.py similarity index 100% rename from monitoring/api/labeled_custom_metric_test.py rename to monitoring/api/v2/labeled_custom_metric_test.py diff --git a/monitoring/api/lightweight_custom_metric.py b/monitoring/api/v2/lightweight_custom_metric.py similarity index 100% rename from monitoring/api/lightweight_custom_metric.py rename to monitoring/api/v2/lightweight_custom_metric.py diff --git a/monitoring/api/lightweight_custom_metric_test.py b/monitoring/api/v2/lightweight_custom_metric_test.py similarity index 100% rename from monitoring/api/lightweight_custom_metric_test.py rename to monitoring/api/v2/lightweight_custom_metric_test.py diff --git a/monitoring/api/requirements.txt b/monitoring/api/v2/requirements.txt similarity index 100% rename from monitoring/api/requirements.txt rename to monitoring/api/v2/requirements.txt diff --git a/monitoring/api/v3/README.md b/monitoring/api/v3/README.md new file mode 100644 index 00000000000..72797754045 --- /dev/null +++ b/monitoring/api/v3/README.md @@ -0,0 +1,47 @@ +# Cloud Monitoring v3 Sample + +Sample command-line programs for retrieving Google Monitoring API V3 data. + +`list_resources.py` is a simple command-line program to demonstrate connecting to the Google +Monitoring API to retrieve API data and print out some of the resources. + +`custom_metric.py` demonstrates how to create a custom metric and write a TimeSeries +value to it. + +## Prerequisites to run locally: + +* [pip](https://pypi.python.org/pypi/pip) + +Go to the [Google Cloud Console](https://console.cloud.google.com). + +* Go to API Manager -> Credentials +* Click 'New Credentials', and create a Service Account or [click here](https://console.cloud.google +.com/project/_/apiui/credential/serviceaccount) + Download the JSON for this service account, and set the `GOOGLE_APPLICATION_CREDENTIALS` + environment variable to point to the file containing the JSON credentials. + + + export GOOGLE_APPLICATION_CREDENTIALS=~/Downloads/-0123456789abcdef.json + + +# Set Up Your Local Dev Environment +To install, run the following commands. If you want to use [virtualenv](https://virtualenv.readthedocs.org/en/latest/) +(recommended), run the commands within a virtualenv. + + * pip install -r requirements.txt + +To run locally: + + python list_resources.py --project_id= + python custom_metric.py --project_id= + +""" + +# [START all] +import argparse +import datetime +import pprint +import random +import time + +import list_resources + + +def format_rfc3339(datetime_instance=None): + """Formats a datetime per RFC 3339. + :param datetime_instance: Datetime instanec to format, defaults to utcnow + """ + return datetime_instance.isoformat("T") + "Z" + + +def get_start_time(): + # Return now- 5 minutes + start_time = datetime.datetime.utcnow() - datetime.timedelta(minutes=5) + return format_rfc3339(start_time) + + +def get_now_rfc3339(): + # Return now + return format_rfc3339(datetime.datetime.utcnow()) + + +def create_custom_metric(client, project_id, + custom_metric_name, metric_kind): + """Create custom metric descriptor""" + metrics_descriptor = { + "name": "projects/{}/metricDescriptors/{}".format( + project_id, custom_metric_name), + "type": custom_metric_name, + "labels": [ + { + "key": "environment", + "valueType": "STRING", + "description": "An abritrary measurement" + } + ], + "metricKind": metric_kind, + "valueType": "INT64", + "unit": "items", + "description": "An arbitrary measurement.", + "displayName": "Custom Metric" + } + + client.projects().metricDescriptors().create( + name=project_id, body=metrics_descriptor).execute() + + +def get_custom_metric(client, project_id, custom_metric_name): + """Retrieve the custom metric we created""" + request = client.projects().metricDescriptors().list( + name=project_id, + filter='metric.type=starts_with("{}")'.format(custom_metric_name)) + response = request.execute() + print('ListCustomMetrics response:') + pprint.pprint(response) + try: + return response['metricDescriptors'] + except KeyError: + return None + + +def get_custom_data_point(): + """Dummy method to return a mock measurement for demonstration purposes. + Returns a random number between 0 and 10""" + length = random.randint(0, 10) + print("reporting timeseries value {}".format(str(length))) + return length + + +def write_timeseries_value(client, project_resource, + custom_metric_name, instance_id, metric_kind): + """Write the custom metric obtained by get_custom_data_point at a point in + time.""" + # Specify a new data point for the time series. + now = get_now_rfc3339() + timeseries_data = { + "metric": { + "type": custom_metric_name, + "labels": { + "environment": "STAGING" + } + }, + "resource": { + "type": 'gce_instance', + "labels": { + 'instance_id': instance_id, + 'zone': 'us-central1-f' + } + }, + "metricKind": metric_kind, + "valueType": "INT64", + "points": [ + { + "interval": { + "startTime": now, + "endTime": now + }, + "value": { + "int64Value": get_custom_data_point() + } + } + ] + } + + request = client.projects().timeSeries().create( + name=project_resource, body={"timeSeries": [timeseries_data]}) + request.execute() + + +def read_timeseries(client, project_resource, custom_metric_name): + """Reads all of the CUSTOM_METRICS that we have written between START_TIME + and END_TIME + :param project_resource: Resource of the project to read the timeseries + from. + :param custom_metric_name: The name of the timeseries we want to read. + """ + request = client.projects().timeSeries().list( + name=project_resource, + filter='metric.type="{0}"'.format(custom_metric_name), + pageSize=3, + interval_startTime=get_start_time(), + interval_endTime=get_now_rfc3339()) + response = request.execute() + return response + + +def main(project_id): + # This is the namespace for all custom metrics + CUSTOM_METRIC_DOMAIN = "custom.googleapis.com" + # This is our specific metric name + CUSTOM_METRIC_NAME = "{}/custom_measurement".format(CUSTOM_METRIC_DOMAIN) + INSTANCE_ID = "test_instance" + METRIC_KIND = "GAUGE" + + project_resource = "projects/{0}".format(project_id) + client = list_resources.get_client() + create_custom_metric(client, project_resource, + CUSTOM_METRIC_NAME, METRIC_KIND) + custom_metric = None + while not custom_metric: + # wait until it's created + time.sleep(1) + custom_metric = get_custom_metric( + client, project_resource, CUSTOM_METRIC_NAME) + + write_timeseries_value(client, project_resource, + CUSTOM_METRIC_NAME, INSTANCE_ID, METRIC_KIND) + # Sometimes on new metric descriptors, writes have a delay in being read + # back. 3 seconds should be enough to make sure our read call picks up the + # write + time.sleep(3) + timeseries = read_timeseries(client, project_resource, CUSTOM_METRIC_NAME) + print('read_timeseries response:\n{}'.format(pprint.pformat(timeseries))) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + '--project_id', help='Project ID you want to access.', required=True) + + args = parser.parse_args() + main(args.project_id) + +# [END all] diff --git a/monitoring/api/v3/custom_metric_test.py b/monitoring/api/v3/custom_metric_test.py new file mode 100644 index 00000000000..7f3a3ecea9a --- /dev/null +++ b/monitoring/api/v3/custom_metric_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Integration test for custom_metric.py + +GOOGLE_APPLICATION_CREDENTIALS must be set to a Service Account for a project +that has enabled the Monitoring API. + +Currently the TEST_PROJECT_ID is hard-coded to run using the project created +for this test, but it could be changed to a different project. +""" + +import random +import time + +from custom_metric import create_custom_metric, get_custom_metric +from custom_metric import read_timeseries, write_timeseries_value +import list_resources + +""" Change this to run against other prjoects + GOOGLE_APPLICATION_CREDENTIALS must be the service account for this project +""" + +# temporarily hard code to whitelisted project +TEST_PROJECT_ID = 'cloud-monitoring-dev' +# TEST_PROJECT_ID = os.getenv("GCLOUD_PROJECT", 'cloud-monitoring-dev') + +""" Custom metric domain for all cusotm metrics""" +CUSTOM_METRIC_DOMAIN = "custom.googleapis.com" + +PROJECT_RESOURCE = "projects/{}".format(TEST_PROJECT_ID) + +METRIC = 'compute.googleapis.com/instance/cpu/usage_time' +METRIC_NAME = ''.join( + random.choice('0123456789ABCDEF') for i in range(16)) +METRIC_RESOURCE = "{}/{}".format( + CUSTOM_METRIC_DOMAIN, METRIC_NAME) + + +def test_custom_metric(): + client = list_resources.get_client() + # Use a constant seed so psuedo random number is known ahead of time + random.seed(1) + pseudo_random_value = random.randint(0, 10) + # Reseed it + random.seed(1) + + INSTANCE_ID = "test_instance" + METRIC_KIND = "GAUGE" + + create_custom_metric( + client, PROJECT_RESOURCE, METRIC_RESOURCE, METRIC_KIND) + custom_metric = None + # wait until metric has been created, use the get call to wait until + # a response comes back with the new metric + while not custom_metric: + time.sleep(1) + custom_metric = get_custom_metric( + client, PROJECT_RESOURCE, METRIC_RESOURCE) + + write_timeseries_value(client, PROJECT_RESOURCE, + METRIC_RESOURCE, INSTANCE_ID, + METRIC_KIND) + # Sometimes on new metric descriptors, writes have a delay in being + # read back. 3 seconds should be enough to make sure our read call + # picks up the write + time.sleep(3) + response = read_timeseries(client, PROJECT_RESOURCE, METRIC_RESOURCE) + value = int( + response['timeSeries'][0]['points'][0]['value']['int64Value']) + # using seed of 1 will create a value of 1 + assert value == pseudo_random_value diff --git a/monitoring/api/v3/list_resources.py b/monitoring/api/v3/list_resources.py new file mode 100644 index 00000000000..88a765fcac9 --- /dev/null +++ b/monitoring/api/v3/list_resources.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Sample command-line program for retrieving Google Monitoring API V3 data. + +See README.md for instructions on setting up your development environment. + +To run locally: + + python list_resources.py --project_id= + +""" + +# [START all] +import argparse +import datetime +import pprint + +from apiclient import discovery +from oauth2client.client import GoogleCredentials + + +def format_rfc3339(datetime_instance): + """Formats a datetime per RFC 3339.""" + return datetime_instance.isoformat("T") + "Z" + + +def get_start_time(): + """ Returns the start time for the 5-minute window to read the custom + metric from within. + :return: The start time to begin reading time series values, picked + arbitrarily to be an hour ago and 5 minutes + """ + # Return an hour ago - 5 minutes + start_time = (datetime.datetime.utcnow() - + datetime.timedelta(hours=1, minutes=5)) + return format_rfc3339(start_time) + + +def get_end_time(): + """ Returns the end time for the 5-minute window to read the custom metric + from within. + :return: The start time to begin reading time series values, picked + arbitrarily to be an hour ago, or 5 minutes from the start time. + """ + end_time = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + return format_rfc3339(end_time) + + +def list_monitored_resource_descriptors(client, project_resource): + """Query the projects.monitoredResourceDescriptors.list API method. + This lists all the resources available to be monitored in the API. + """ + request = client.projects().monitoredResourceDescriptors().list( + name=project_resource) + response = request.execute() + print('list_monitored_resource_descriptors response:\n{}'.format( + pprint.pformat(response))) + + +def list_metric_descriptors(client, project_resource, metric): + """Query to MetricDescriptors.list + This lists the metric specified by METRIC. + """ + request = client.projects().metricDescriptors().list( + name=project_resource, + filter='metric.name="{}"'.format(metric)) + response = request.execute() + print( + 'list_metric_descriptors response:\n{}'.format( + pprint.pformat(response))) + + +def list_timeseries(client, project_resource, metric): + """Query the TimeSeries.list API method. + This lists all the timeseries created between START_TIME and END_TIME. + """ + request = client.projects().timeSeries().list( + name=project_resource, + filter='metric.name="{}"'.format(metric), + pageSize=3, + interval_startTime=get_start_time(), + interval_endTime=get_end_time()) + response = request.execute() + print('list_timeseries response:\n{}'.format(pprint.pformat(response))) + + +def get_client(): + """Builds an http client authenticated with the service account + credentials.""" + credentials = GoogleCredentials.get_application_default() + client = discovery.build('monitoring', 'v3', credentials=credentials) + return client + + +def main(project_id): + project_resource = "projects/{}".format(project_id) + client = get_client() + list_monitored_resource_descriptors(client, project_resource) + # Metric to list + metric = 'compute.googleapis.com/instance/cpu/usage_time' + list_metric_descriptors(client, project_resource, metric) + list_timeseries(client, project_resource, metric) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + '--project_id', help='Project ID you want to access.', required=True) + + args = parser.parse_args() + main(args.project_id) + +# [END all] diff --git a/monitoring/api/v3/list_resources_test.py b/monitoring/api/v3/list_resources_test.py new file mode 100644 index 00000000000..ed7b8f43706 --- /dev/null +++ b/monitoring/api/v3/list_resources_test.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Integration test for list_env.py + +GOOGLE_APPLICATION_CREDENTIALS must be set to a Service Account for a project +that has enabled the Monitoring API. + +Currently the TEST_PROJECT_ID is hard-coded to run using the project created +for this test, but it could be changed to a different project. +""" + +import re + +import list_resources + +# temporarily hard code to whitelisted project +TEST_PROJECT_ID = 'cloud-monitoring-dev' +# TEST_PROJECT_ID = os.getenv("GCLOUD_PROJECT", 'cloud-monitoring-dev') +PROJECT_RESOURCE = "projects/{}".format(TEST_PROJECT_ID) +METRIC = 'compute.googleapis.com/instance/cpu/usage_time' + + +def test_list_monitored_resources(capsys): + client = list_resources.get_client() + list_resources.list_monitored_resource_descriptors( + client, PROJECT_RESOURCE) + stdout, _ = capsys.readouterr() + regex = re.compile( + 'An application running') + assert regex.search(stdout) is not None + + +def test_list_metrics(capsys): + client = list_resources.get_client() + list_resources.list_metric_descriptors( + client, PROJECT_RESOURCE, METRIC) + stdout, _ = capsys.readouterr() + regex = re.compile( + u'Delta CPU usage time') + assert regex.search(stdout) is not None + + +def test_list_timeseries(capsys): + client = list_resources.get_client() + list_resources.list_timeseries( + client, PROJECT_RESOURCE, METRIC) + stdout, _ = capsys.readouterr() + regex = re.compile(u'list_timeseries response:\n') + assert regex.search(stdout) is not None diff --git a/monitoring/api/v3/requirements.txt b/monitoring/api/v3/requirements.txt new file mode 100644 index 00000000000..ddf776e3c10 --- /dev/null +++ b/monitoring/api/v3/requirements.txt @@ -0,0 +1,10 @@ +google-api-python-client==1.5.0 +httplib2==0.9.2 +oauth2client==1.5.1 +pyasn1==0.1.9 +pyasn1-modules==0.0.8 +rsa==3.2.3 +simplejson==3.8.1 +six==1.10.0 +uritemplate==0.6 +wheel==0.24.0