diff --git a/README.md b/README.md index 18994e3..4708f93 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,28 @@ cloudwatch-mon-scripts-python [![PyPI version](https://badge.fury.io/py/cloudwatchmon.svg)](https://badge.fury.io/py/cloudwatchmon) -Linux monitoring scripts for CloudWatch. +Linux monitoring scripts for the [AWS CloudWatch Service](https://aws.amazon.com/de/cloudwatch/). + +Initially, this project was created, because the +original [AWS monitoring scripts](https://aws.amazon.com/code/8720044071969977) +lacked support for the eu-central-1 (Frankfurt) region for about 4 months +(2014-10-23 to 2015-02-25). + +Now, this project has a couple of additional features (compared to v1.1.0 of +the original AWS monitoring scripts): + +- Memory monitoring incl. buffers +- Load monitoring (overall and per CPU core) +- Monitoring of disk inode usage +- Process monitoring +- Fewer dependencies +- Simpler installation Requirements ------------ -- Python >= 2.6 +- Python 2 (>= 2.6) or Python 3 (>= 3.3) - Boto >= 2.33.0 @@ -17,7 +32,7 @@ Installation ------------ Optionally create a virtual environment and activate it. Then just run -`pip install cloudwatchmon`. Install the scripts in /usr/local/bin folder. +`pip install cloudwatchmon`. Install the scripts in `/usr/local/bin` folder. For script usage, run: @@ -27,15 +42,15 @@ For script usage, run: Examples -------- -To perform a simple test run without posting data to Amazon CloudWatch +To perform a simple test run without posting data to Amazon CloudWatch: mon-put-instance-stats.py --mem-util --verify --verbose -Report memory and disk space utilization to Amazon CloudWatch +Report memory and disk space utilization to Amazon CloudWatch: mon-put-instance-stats.py --mem-util --disk-space-util --disk-path=/ -To get utilization statistics for the last 12 hours +To get utilization statistics for the last 12 hours: mon-get-instance-stats.py --recent-hours=12 @@ -46,20 +61,22 @@ Configuration To allow an EC2 instance to read and post metric data to Amazon CloudWatch, this IAM policy is required: +```json +{ + "Statement": [ { - "Statement": [ - { - "Action": [ - "cloudwatch:ListMetrics", - "cloudwatch:GetMetricStatistics", - "cloudwatch:PutMetricData", - "autoscaling:DescribeAutoScalingInstances" - ], - "Effect": "Allow", - "Resource": "*" - } - ] + "Action": [ + "cloudwatch:ListMetrics", + "cloudwatch:GetMetricStatistics", + "cloudwatch:PutMetricData", + "autoscaling:DescribeAutoScalingInstances" + ], + "Effect": "Allow", + "Resource": "*" } + ] +} +``` If the policy is configured via an IAM role that is assigned to the EC2 server this script runs on, you're done. @@ -67,15 +84,18 @@ server this script runs on, you're done. Otherwise you can configure the policy for a user account and export the credentials before running the script: - export AWS_ACCESS_KEY_ID=[Your AWS Access Key ID] - export AWS_SECRET_ACCESS_KEY=[Your AWS Secret Access Key] +```sh +export AWS_ACCESS_KEY_ID=[Your AWS Access Key ID] +export AWS_SECRET_ACCESS_KEY=[Your AWS Secret Access Key] +``` Third option is to create a _~/.boto_ file with this content: - [Credentials] - aws_access_key_id = Your AWS Access Key ID - aws_secret_access_key = Your AWS Secret Access Key - +``` +[Credentials] +aws_access_key_id = Your AWS Access Key ID +aws_secret_access_key = Your AWS Secret Access Key +``` Copyright --------- diff --git a/cloudwatchmon/__init__.py b/cloudwatchmon/__init__.py index fa8968b..780e307 100644 --- a/cloudwatchmon/__init__.py +++ b/cloudwatchmon/__init__.py @@ -1 +1 @@ -VERSION = '2.0.5' +VERSION = '2.0.7' diff --git a/cloudwatchmon/cli/get_instance_stats.py b/cloudwatchmon/cli/get_instance_stats.py index 93d3404..6e800e6 100755 --- a/cloudwatchmon/cli/get_instance_stats.py +++ b/cloudwatchmon/cli/get_instance_stats.py @@ -71,7 +71,7 @@ def print_metric_stats(region, instance_id, namespace, metric, title, start_time = end_time - datetime.timedelta(hours=recent_hours) dims = {'InstanceId': instance_id} if xdims: - dims = dict(dims.items() + xdims.items()) + dims.update(xdims) metrics = conn.get_metric_statistics(300, start_time, end_time, metric, namespace, ['Average', 'Maximum', 'Minimum'], diff --git a/cloudwatchmon/cli/put_instance_stats.py b/cloudwatchmon/cli/put_instance_stats.py index 9e30e41..bf83721 100755 --- a/cloudwatchmon/cli/put_instance_stats.py +++ b/cloudwatchmon/cli/put_instance_stats.py @@ -50,6 +50,7 @@ def __init__(self, mem_used_incl_cache_buff): mem_info = self.__gather_mem_info() self.mem_total = mem_info['MemTotal'] self.mem_free = mem_info['MemFree'] + self.mem_available = mem_info.get('MemAvailable') # Linux >= 3.14 self.mem_cached = mem_info['Cached'] self.mem_buffers = mem_info['Buffers'] self.swap_total = mem_info['SwapTotal'] @@ -74,9 +75,14 @@ def mem_used(self): return self.mem_total - self.mem_avail() def mem_avail(self): - mem_avail = self.mem_free - if not self.mem_used_incl_cache_buff: - mem_avail += self.mem_cached + self.mem_buffers + # Linux 3.14 kernel added 'MemAvailable' to /proc/meminfo + if self.mem_available: + mem_avail = self.mem_available + else: + mem_avail = self.mem_free + self.mem_cached + self.mem_buffers + + if self.mem_used_incl_cache_buff: + mem_avail -= self.mem_cached + self.mem_buffers return mem_avail @@ -170,13 +176,15 @@ def __add_metric_dimensions(self, name, unit, value, common_dims, dims): self.names.append(name) self.units.append(unit) self.values.append(value) - self.dimensions.append(dict(common_dims.items() + dim.items())) + metric_dims = common_dims.copy() + metric_dims.update(dim) + self.dimensions.append(metric_dims) - def send(self, verbose): + def send(self, verbose, awsProfile): boto_debug = 2 if verbose else 0 # TODO add timeout - conn = boto.ec2.cloudwatch.connect_to_region(self.region, + conn = boto.ec2.cloudwatch.connect_to_region(self.region, profile_name=awsProfile, debug=boto_debug) if not conn: @@ -222,23 +230,20 @@ def config_parser(): Examples - To perform a simple test run without posting data to Amazon CloudWatch + To perform a simple test run without posting data to Amazon CloudWatch + + mon-put-instance-stats.py --mem-util --verify --verbose - ./put_instance_stats.py --mem-util --verify --verbose - or - # If installed via pip install cloudwatchmon - mon-put-instance-stats.py --mem-util --verify --verbose - To set a five-minute cron schedule to report memory and disk space utilization - to CloudWatch + To set a five-minute cron schedule to report memory and disk space utilization + to CloudWatch + + */5 * * * * mon-put-instance-stats.py --mem-util --disk-space-util --disk-path=/ --from-cron - */5 * * * * ~/cloudwatchmon/put_instance_stats.py --mem-util --disk-space-util --disk-path=/ --from-cron - or - # If installed via pip install cloudwatchmon - * /5 * * * * /usr/local/bin/mon-put-instance-stats.py --mem-util --disk-space-util --disk-path=/ --from-cron To report metrics from file - mon-put-instance-stats.py --from-file filename.csv + + mon-put-instance-stats.py --from-file filename.csv For more information on how to use this utility, see project home on GitHub: https://github.com/osiegmar/cloudwatch-mon-scripts-python @@ -311,7 +316,7 @@ def config_parser(): process_group.add_argument('--process-name', metavar='PROCNAME', action='append', - help='Report CPU and Memory utilization metrics of processes.') + help='Report process count, CPU utilization, and memory utilization metrics for a process.') exclusive_group = parser.add_mutually_exclusive_group() exclusive_group.add_argument('--from-cron', @@ -339,6 +344,9 @@ def config_parser(): parser.add_argument('--version', action='store_true', help='Displays the version number and exits.') + parser.add_argument('--aws-profile-name', + action='store', + help='Use AWS profile of this name when posting to CloudWatch.') return parser @@ -433,13 +441,16 @@ def add_process_metrics(args, metrics): process_names = args.process_name for process_name in process_names: processes = subprocess.Popen(["ps", "axco", "command,pcpu,pmem"], stdout=subprocess.PIPE) + total_cnt = 0 total_cpu = 0.0 total_mem = 0.0 for line in processes.stdout: if re.search(process_name, line): + total_cnt += 1 out = line.split() total_cpu += float(out[1]) total_mem += float(out[2]) + metrics.add_metric(process_name+'-Count', 'Count', total_cnt) metrics.add_metric(process_name+'-CpuUtilization', 'Percent', total_cpu) metrics.add_metric(process_name+'-MemoryUtilization', 'Percent', total_mem) @@ -496,7 +507,8 @@ def validate_args(args): 'disk path is not specified.') if not report_mem_data and not report_disk_data and \ - not args.from_file and not report_loadavg_data: + not args.from_file and not report_loadavg_data and \ + not report_process_data: raise ValueError('No metrics specified for collection and ' 'submission to CloudWatch.') @@ -570,12 +582,17 @@ def main(): if args.verbose: print('Request:\n' + str(metrics)) + # Check for AWS profile + awsProfile = None + if args.aws_profile_name: + awsProfile = args.aws_profile_name + if args.verify: if not args.from_cron: print('Verification completed successfully. ' 'No actual metrics sent to CloudWatch.') else: - metrics.send(args.verbose) + metrics.send(args.verbose, awsProfile) if not args.from_cron: print('Successfully reported metrics to CloudWatch.') except Exception as e: diff --git a/cloudwatchmon/cloud_watch_client.py b/cloudwatchmon/cloud_watch_client.py index bee4e99..e564e18 100644 --- a/cloudwatchmon/cloud_watch_client.py +++ b/cloudwatchmon/cloud_watch_client.py @@ -44,9 +44,9 @@ def __init__(self, fnc): def __call__(self, *args, **kwargs): sig = ":".join([VERSION, str(self.fnc.__name__), str(args), str(kwargs)]) + sig_hash = hashlib.md5(sig.encode('utf-8')).hexdigest() filename = os.path.join(META_DATA_CACHE_DIR, '{0}-{1}.bin' - .format(self.CLIENT_NAME, - hashlib.md5(sig).hexdigest())) + .format(self.CLIENT_NAME, sig_hash)) if os.path.exists(filename): mtime = os.path.getmtime(filename) diff --git a/setup.py b/setup.py index 2361d0b..80739d9 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def readme(): long_description=readme(), url='https://github.com/osiegmar/cloudwatch-mon-scripts-python', author='Oliver Siegmar', - author_email='oliver@siegmar.net', + author_email='oliver@siegmar.de', license='Apache License (2.0)', keywords="monitoring cloudwatch amazon web services aws ec2", zip_safe=True, @@ -30,8 +30,19 @@ def readme(): ] }, classifiers=[ - 'Programming Language :: Python :: 2 :: Only', + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: System :: Monitoring' ] )