Add Logging API Samples

Bill Prin · Bill Prin · commit 2006c55b8e17 · 2016-06-13T15:47:40.000-07:00
diff --git a/logging/api/README.md b/logging/api/README.md
@@ -0,0 +1,64 @@
+# Cloud Logging v2 API Samples
+
+Sample command-line programs for retrieving Google Logging API V2 data.
+
+`logs_api.py` is a simple command-line program to demonstrate writing to a log,
+listing its entries to view it, then deleting it via CLI operations.
+
+`export_logs_api.py` demonstrates how to interact with Logging sinks, which can send
+logs to Google Cloud Storage, Cloud Pub/Sub, or BigQuery. In this example
+we use Google Cloud Storage. It similarly exposes a CLI to run these operations.
+
+## Prerequisites to run locally:
+
+
+* A Google Cloud Project
+
+Go to the [Google Cloud Console](https://console.cloud.google.com) to create
+ a project. Take note of the project ID, which is sometimes but not always
+ the same as your given name for a project.
+
+To run `export.py`, you will also need a Google Cloud Storage Bucket. 
+
+    gsutil mb gs://[YOUR_PROJECT_ID]
+
+You must add Cloud Logging as an owner to the bucket. To do so, add cloud-logs@google.com as
+an owner to the bucket. See the [exportings logs](https://cloud.google.com/logging/docs/export/configure_export#configuring_log_sinks)
+docs for complete details. 
+
+# Set Up Your Local Dev Environment
+To install, run the following commands. If you want to use  [virtualenv](https://virtualenv.readthedocs.org/en/latest/)
+(recommended), run the commands within a virtualenv.
+
+Create local credentials by running the following command and following the oauth2 flow:
+
+    gcloud beta auth application-default login
+
+To run the list_logs example
+
+    python logs_api.py --project_id=<YOUR-PROJECT-ID> write_entry "hello world"
+    python logs_api.py --project_id=<YOUR-PROJECT-ID> list_entries
+    python logs_api.py --project_id=<YOUR-PROJECT-ID> delete_logger
+    
+
+The `exports_logs_api.py` samples requires a Cloud bucket that has added cloud-logs@google.com 
+as an owner. See:
+
+https://cloud.google.com/logging/docs/export/configure_export#setting_product_name_short_permissions_for_writing_exported_logs
+    
+    python export_logs_api.py --project_id=YOUR_PROJECT_ID create_sink \  
+        --destination_bucket=YOUR_BUCKET
+        
+    python export_logs_api.py --project_id=YOUR_PROJECT_ID update_sink\  
+        --destination_bucket=YOUR_BUCKET
+                
+    python export_logs_api.py --project_id=YOUR_PROJECT_ID list_sinks
+              
+    python export_logs_api.py --project_id=YOUR_PROJECT_ID delete_sink \  
+              --destination_bucket=YOUR_BUCKET
+
+
+## Running on GCE, GAE, or other environments
+
+See our [Cloud Platform authentication guide](https://cloud.google.com/docs/authentication).
+
diff --git a/logging/api/export_logs_api.py b/logging/api/export_logs_api.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+from gcloud import logging
+from oauth2client.client import GoogleCredentials
+
+FILTER = 'logName="projects/{}/logs/syslog" AND severity>=ERROR'
+DESTINATION = 'storage.googleapis.com/{}'
+
+
+def create_sink_if_not_exists(client, args):
+    # [START create]
+    sink = client.sink(
+        args.sink_name,
+        FILTER.format(args.project_id),
+        DESTINATION.format(args.destination_bucket))
+
+    if not sink.exists():
+        sink.create()
+        print('Created sink {}'.format(sink.name))
+    # [END create]
+    return sink
+
+
+def list_sinks(client, args):
+    print('Listing sinks available')
+
+    # [START list]
+    sinks = []
+    while True:
+        new_sinks, token = client.list_sinks()
+        sinks += new_sinks
+        if token is None:
+            break
+
+    for sink in sinks:
+        print('{}: {}'.format(sink.name, sink.destination))
+    # [END list]
+
+    return sinks
+
+
+def update_sink(client, args):
+    """Changes the filter of a sink.
+
+     The filter is used to determine which log statements match this sink and
+     will be exported to the destination.
+    """
+    # Removes the robot in textPayload part of filter
+    sink = client.sink(
+        args.sink_name,
+        FILTER.format(args.project_id),
+        DESTINATION.format(args.destination_bucket))
+    # [START update]
+    sink.filter = ('logName="projects/{}/logs/syslog" '
+                   'AND severity>= INFO'.format(sink.project))
+    print('Updated sink {}'.format(sink.name))
+    sink.update()
+    # [END update]
+
+
+def delete_sink(client, args):
+    """Deletes a sink"""
+    sink = client.sink(
+        args.sink_name,
+        FILTER.format(args.project_id),
+        DESTINATION.format(args.destination_bucket))
+    # [START delete]
+    sink.delete()
+    # [END delete]
+    print('Deleted sink {}'.format(sink.name))
+
+
+def get_client(project_id):
+    """Builds an http client authenticated with the service account
+    credentials."""
+    credentials = GoogleCredentials.get_application_default()
+    return logging.Client(project=project_id, credentials=credentials)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        '--project_id', help='Project ID you want to access.', required=True,)
+
+    parser.add_argument(
+        '--sink_name', help='Output bucket to direct sink to',
+        default="mysink")
+
+    subparsers = parser.add_subparsers()
+
+    create_parser = subparsers.add_parser('create_sink')
+    create_parser.set_defaults(func=create_sink_if_not_exists)
+    create_parser.add_argument(
+        '--destination_bucket', help='Output bucket to direct sink to',
+        required=True)
+
+    list_parser = subparsers.add_parser('list_sinks')
+    list_parser.set_defaults(func=list_sinks)
+
+    update_parser = subparsers.add_parser('update_sink')
+    update_parser.set_defaults(func=update_sink)
+    update_parser.add_argument(
+        '--destination_bucket', help='Output bucket to direct sink to',
+        required=True)
+
+    delete_parser = subparsers.add_parser('delete_sink')
+    delete_parser.add_argument(
+        '--destination_bucket', help='Output bucket to direct sink to',
+        required=True)
+    delete_parser.set_defaults(func=delete_sink)
+
+    args = parser.parse_args()
+    client = get_client(args.project_id)
+    args.func(client, args)
diff --git a/logging/api/export_logs_api_test.py b/logging/api/export_logs_api_test.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+
+import export_logs_api
+
+SINK_NAME = 'test_sink'
+
+
+def test_sinks(cloud_config):
+    client = export_logs_api.get_client(cloud_config.project)
+
+    Args = namedtuple('Args', 'project_id destination_bucket sink_name')
+    args = Args(
+        destination_bucket=cloud_config.storage_bucket,
+        project_id=cloud_config.project,
+        sink_name=SINK_NAME)
+
+    export_logs_api.create_sink_if_not_exists(client, args)
+    sinks = export_logs_api.list_sinks(client, args)
+    matched_sinks = [s for s in sinks if s.name == SINK_NAME]
+    assert len(matched_sinks) == 1
+    export_logs_api.update_sink(client, args)
+    export_logs_api.delete_sink(client, args)
diff --git a/logging/api/logs_api.py b/logging/api/logs_api.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+from gcloud import logging
+from oauth2client.client import GoogleCredentials
+
+
+def write_entry(client, args):
+    print('Writing log entry for logger '.format(args.logger_name))
+    mylogger = client.logger(args.logger_name)
+    # [START write]
+    mylogger.log_text(args.entry)
+    # [END write]
+
+
+def list_entries(client, args):
+    """Lists all entries for a logger"""
+    logger = client.logger(args.logger_name)
+    print('Listing all log entries for logger {}'.format(logger.name))
+    # [START list]
+    entries = []
+    while True:
+        new_entries, token = client.list_entries(filter_='logName="{}"'.format(
+           logger.full_name))
+        entries += new_entries
+        if token is None:
+            break
+
+    for entry in entries:
+        timestamp = entry.timestamp.isoformat()
+        print('{}: {}'.format
+              (timestamp, entry.payload))
+    # [END list]
+    return entries
+
+
+def delete_logger(client, args):
+    """Deletes a logger and all its entries.
+
+    Note that a deletion can take several minutes to take effect.
+    """
+    logger = client.logger(args.logger_name)
+    print('Deleting all logging entries for {}'.format(logger.name))
+    # [START delete]
+    logger.delete()
+    # [END delete]
+
+
+def get_client(project_id):
+    """Builds an http client authenticated with the service account
+    credentials."""
+    # [START auth]
+    credentials = GoogleCredentials.get_application_default()
+    return logging.Client(project=project_id, credentials=credentials)
+    # [END auth]
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        '--project_id', help='Project ID you want to access.', required=True)
+    parser.add_argument(
+        '--logger_name', help='Logger name', default='mylogger')
+
+    subparsers = parser.add_subparsers()
+
+    write_parser = subparsers.add_parser('write_entry')
+    write_parser.add_argument('entry')
+    write_parser.set_defaults(func=write_entry)
+
+    list_parser = subparsers.add_parser('list_entries')
+    list_parser.set_defaults(func=list_entries)
+
+    delete_parser = subparsers.add_parser('delete_logger')
+    delete_parser.set_defaults(func=delete_logger)
+
+    args = parser.parse_args()
+    client = get_client(args.project_id)
+    args.func(client, args)
diff --git a/logging/api/logs_api_test.py b/logging/api/logs_api_test.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+import time
+
+from gcp.testing.flaky import flaky
+import logs_api
+
+
+@flaky
+def test_logs(cloud_config):
+    client = logs_api.get_client(cloud_config.project)
+
+    LOG_MESSAGE = 'hello world'
+    Args = namedtuple('Args', 'logger_name entry')
+    args = Args(logger_name='test_log', entry=LOG_MESSAGE)
+
+    logs_api.write_entry(client, args)
+    time.sleep(3)
+    entries = logs_api.list_entries(client, args)
+    matched_entries = [e for e in entries if e.payload == LOG_MESSAGE]
+    assert len(matched_entries) > 0
+
+    logs_api.delete_logger(client, args)
diff --git a/logging/api/requirements.txt b/logging/api/requirements.txt
@@ -0,0 +1 @@
+gcloud==0.15.0
