From 39f6907ffa12e49adda47c3cb67b8480182a9681 Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:27:34 +0530 Subject: [PATCH 01/18] Major Refactoring --- linode.py | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 linode.py diff --git a/linode.py b/linode.py new file mode 100644 index 0000000..f6f978c --- /dev/null +++ b/linode.py @@ -0,0 +1,123 @@ +import os +import random +import functools +from base import ProxyRotator +from utils import randpass + +class LinodeCommand(object): + """ Class encapsulating linode CLI commands """ + + def __init__(self, binary='linode', verbose=False, config=None): + self.binary = binary + self.verbose = verbose + self.cmd_template = {'create': 'create -d %d -p %d -o %d -i %d -l %s -r %s', + 'delete': 'delete -l %d', + 'list_proxies': 'find -g %s -s %s' % (config.group, config.proxylb), + 'info': 'info -l %d', + 'update': 'update -l %d -L %s -g %s' + } + # Dynamically create command methods + self.dyn_create() + + def _run(self, command, *args): + """ Run a command and return the output """ + + template = self.cmd_template.get(command) + if template == None: + print 'No such command configured =>',command + return -1 + + cmd = ' '.join((self.binary, template % args)) + if self.verbose: print 'Command is',cmd + return os.popen(cmd).read() + + def dyn_create(self): + """ Dynamically create linode methods """ + + for cmd in self.cmd_template: + method_name = cmd + method = functools.partial(self._run, cmd) + if self.verbose: print 'Dyn-creating method',method_name,'...' + setattr(self, method_name, method) + + def get_label(self, linode_id): + """ Return the label, given the linode id """ + + data = self.info(linode_id) + return data.split('\n')[0].split(':')[-1].strip() + + def get_proxies(self): + """ Return all proxies as a list """ + + return self.list_proxies().strip().split('\n') + +class LinodeProxyRotator(ProxyRotator): + """ Linode VPS implementation of ProxyRotator """ + + def __init__(self, cfg='proxy.conf', test_mode=False, rotate=False, region=None): + super(LinodeProxyRotator, self).__init__(cfg, test_mode, rotate, region) + # Linode creation class + self.linode_command = LinodeCommand(verbose=True, config=self.config) + self.vps_command = self.linode_command + + def get_instance_label(self, instance_id): + """ Return instance label given instance id """ + return self.linode_command.get_label(instance_id) + + def delete_instance(self, instance_id): + """ Delete instance by id """ + return self.linode_command.delete(proxy_out_id) + + def make_new_instance(self, region, test=False, verbose=False): + """ Make a new instance in the given region """ + + # If calling as test, make up an ip + if test: + return '.'.join(map(lambda x: str(random.randrange(20, 100)), range(4))), random.randrange(10000, + 50000) + + tup = (region, + self.config.plan_id, + self.config.os_id, + self.config.image_id, + 'proxy_disk', + randpass()) + + print 'Making new linode in region',region,'...' + data = self.linode_command.create(*tup) + + # data = os.popen(cmd).read() + if verbose: + print data + # The IP is the last line of the command + ip = data.strip().split('\n')[-1].strip().split()[-1].strip() + # Proxy ID + pid = data.strip().split('\n')[-3].strip().split()[-1].strip() + print 'I.P address of new linode is',ip + print 'ID of new linode is',pid + # Post process the host + print 'Post-processing',ip,'...' + self.post_process(ip) + + def update_instance(self, instance_id, label, group=None): + """ Update meta-data for a new instance """ + + # Updates label (name) and group information + ret = self.linode_command.update(int(instance_id), + label, + group) + return ret + + + def drop(self): + """ Drop all the proxies in current configuration (except the LB) """ + + print 'Dropping all proxies ...' + proxies = self.linode_command.get_proxies() + + for item in proxies: + if item.strip() == "": continue + ip,dc,lid,si,so = item.split(',') + print '\tDropping linode',lid,'with IP',ip,'from dc',dc,'...' + self.linode_command.delete(int(lid)) + From 55506af83e20884462f96afa722a7e42ca49e73b Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:27:39 +0530 Subject: [PATCH 02/18] Major Refactoring --- config.py | 268 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..8291b7b --- /dev/null +++ b/config.py @@ -0,0 +1,268 @@ +import json +import time +import random +import operator +import sys +import os +from utils import enum + +# Configurations + +# Rotation Policies +Policy = enum('ROTATION_RANDOM', + # Least recently used + 'ROTATION_LRU', + # Switch to another region + 'ROTATION_NEW_REGION', + # LRU + New region + 'ROTATION_LRU_NEW_REGION') + +region_dict = {2: 'Dallas', + 3: 'Fremont', + 4: 'Atlanta', + 6: 'Newark', + 7: 'London', + 8: 'Tokyo', + 9: 'Singapore', + 10: 'Frankfurt'} + + +email_template = """ + +I just switched a proxy node in the proxy infrastructure. Details are below. + +In: %(label)s, %(proxy_in)s +Out: %(label)s, %(proxy_out)s + +Region: %(region)s + +-- Linode proxy daemon + +""" + +# Post process command +post_process_cmd_template = """ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@%s "%s" """ +iptables_restore_cmd = "sudo iptables-restore < /etc/iptables.rules" +squid_restart_cmd = "sudo squid3 -f /etc/squid3/squid.conf" + + +class ProxyConfig(object): + """ Class representing configuration of crawler proxy infrastructure """ + + def __init__(self, cfg='proxy.conf'): + """ Initialize proxy config from the config file """ + + self.parse_config(cfg) + # This is a file with each line of the form + # IPV4 address, datacenter code, linode-id, switch_in timestamp, switch_out timestamp + # E.g: 45.79.91.191, 3, 1446731065, 144673390 + try: + proxies = map(lambda x: x.strip().split(','), open(self.proxylist).readlines()) + # Proxy IP to (switch_in, switch_out) timestamp mappings + self.proxy_dict = {} + # Proxy IP to enabled mapping + self.proxy_state = {} + self.process_proxies(proxies) + except (OSError, IOError), e: + print e + sys.exit("Fatal error, proxy list input file " + self.proxylist + " not found!") + except ValueError, e: + print e + print self.proxylist + " is empty or has junk values" + + try: + self.proxy_template = open(self.lb_template).read() + except (OSError, IOError), e: + print e + sys.exit("Fatal error, template config input file " + template_file + " not found!") + + def parse_config(self, cfg): + """ Parse the configuration file and load config """ + + self.config = json.load(open(cfg)) + for key,value in self.config.items(): + # Set attribute locally + setattr(self, key, value) + + # Do some further processing + self.frequency = float(self.frequency)*3600.0 + self.policy = eval('Policy.' + self.policy) + + def get_proxy_ips(self): + """ Return all proxy IP addresses as a list """ + + return self.proxy_state.keys() + + def get_active_proxies(self): + """ Return a list of all active proxies as a list """ + + return map(self.proxy_dict.get, filter(self.proxy_state.get, self.proxy_state.keys())) + + def process_proxies(self, proxies): + """ Process the proxy information to create internal dictionaries """ + + # Prepare the proxy region dict + for proxy_ip, region, proxy_id, switch_in, switch_out in proxies: + # If switch_in ==0: put current time + if int(float(switch_in))==0: + switch_in = int(time.time()) + if int(float(switch_out))==0: + switch_out = int(time.time()) + + self.proxy_dict[proxy_ip] = [proxy_ip, int(region), proxy_id, int(float(switch_in)), int(float(switch_out))] + self.proxy_state[proxy_ip] = True + + print 'Processed',len(self.proxy_state),'proxies.' + + def get_proxy_for_rotation(self, + use_random=False, + least_used=False, + region_switch=False, + input_region=3): + """ Return a proxy IP address for rotation using the given settings. The + returned proxy will be replaced with a new proxy. + + @use_random - Means returns a random proxy from the current active list + @least_used - Returns a proxy IP which is the oldest switched out one + so we keep the switching more or less democratic. + @region_switch - Returns a proxy which belongs to a different region + from the new proxy. + @input_region - The region of the new proxy node - defaults to Fremont, CA. + + Note that if use_random is set to true, the other parameters are ignored. + + """ + + active_proxies = self.get_active_proxies() + print 'Active proxies =>',active_proxies + + if use_random: + # Pick a random proxy IP + proxy = random.choice(active_proxies) + print 'Returning proxy =>',proxy + proxy_ip = proxy[0] + + # Remove it from every data structure + self.switch_out_proxy(proxy_ip) + return proxy + + if least_used: + # Pick the oldest switched out proxy i.e one + # with smallest switched out value + proxies_used = sorted(active_proxies, + key=operator.itemgetter(-1)) + + print 'Proxies used =>',proxies_used + + if region_switch: + # Find the one with a different region from input + for proxy, reg, pi, si, so in proxies_used: + if reg != input_region: + print 'Returning proxy',proxy,'from region',reg + self.switch_out_proxy(proxy) + return proxy + + # If all regions are already in use, pick the last used + # proxy anyway + return proxies_used[0][0] + + if region_switch: + # Pick a random proxy not in the input region + proxies = active_proxies + random.shuffle(proxies) + + for proxy, reg, pi, si, so in proxies: + if reg != input_region: + print 'Returning proxy',proxy,'from region',reg + self.switch_out_proxy(proxy) + return proxy + + def __getattr__(self, name): + """ Return from local, else written from config """ + + try: + return self.__dict__[name] + except KeyError: + return self.config.get(name) + + def switch_out_proxy(self, proxy): + """ Switch out a given proxy IP """ + + # Disable it + self.proxy_state[proxy] = False + # Mark its switched out timestamp + self.proxy_dict[proxy][-1] = int(time.time()) + + def switch_in_proxy(self, proxy, proxy_id, region): + """ Switch in a given proxy IP """ + + # Mark its switched out timestamp + self.proxy_dict[proxy] = [proxy, int(region), proxy_id, int(time.time()), int(time.time())] + # Enable it + self.proxy_state[proxy] = True + + def get_active_regions(self): + """ Return unique regions for which proxies are active """ + + regions = set() + for proxy,region,pi,si,so in self.proxy_dict.values(): + if self.proxy_state[proxy]: + regions.add(region) + + return list(regions) + + def write(self, disabled=False): + """ Write current state to an output file """ + + lines = [] + for proxy, reg, pi, si, so in self.proxy_dict.values(): + if disabled or self.proxy_state[proxy]: + lines.append('%s,%s,%s,%s,%s\n' % (proxy, str(reg), str(pi), str(int(si)), str(int(so)))) + + open(self.proxylist,'w').writelines(lines) + + def write_lb_config(self, disabled=False, test=False): + """ Write current proxy configuration into the load balancer config """ + + lines, idx = [], 1 + # Shuffle + items = self.proxy_dict.values() + for i in range(10): + random.shuffle(items) + + for proxy, reg, pi, si, so in items: + if self.proxy_state[proxy]: + lines.append('\tserver squid%d %s:8321 check inter 10000 rise 2 fall 5' % (idx, proxy)) + idx += 1 + + squid_config = "\n".join(lines) + content = self.proxy_template % locals() + # Write to temp file + tmpfile = '/tmp/.haproxy.cfg' + open(tmpfile,'w').write(content) + + # If running in test mode, don't do this! + if not test: + # Run as sudo + cmd = 'sudo cp %s %s; rm -f %s' % (tmpfile, self.lb_config, tmpfile) + os.system(cmd) + + self.reload_lb() + return True + + def reload_lb(self): + """ Reload the HAProxy load balancer """ + + return (os.system(self.lb_restart) == 0) + + def get_proxy_id(self, proxy): + """ Given proxy return its id """ + + return self.proxy_dict[proxy][2] + + def get_email_config(self): + """ Return email configuration """ + + return self.config['email'] + + From 7df8af4891a304243a8b1594a40eb3721640202b Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:27:44 +0530 Subject: [PATCH 03/18] Major Refactoring --- base.py | 281 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 base.py diff --git a/base.py b/base.py new file mode 100644 index 0000000..9f09fb8 --- /dev/null +++ b/base.py @@ -0,0 +1,281 @@ +import os +import threading +import signal +import time +import collections + +import email_report +from config import * + +from utils import daemonize + +class ProxyRotator(object): + """ Proxy rotation, provisioning & re-configuration base class """ + + def __init__(self, cfg='proxy.conf', test_mode=False, rotate=False, region=None): + self.config = ProxyConfig(cfg=cfg) + print 'Frequency set to',self.config.frequency,'seconds.' + # Test mode ? + self.test_mode = test_mode + # Event object + self.alarm = threading.Event() + # Clear the event + self.alarm.clear() + # Heartbeat file + self.hbf = '.heartbeat' + # Actual command used for doing stuff + self.vps_command = None + # If rotate is set, rotate before going to sleep + if rotate: + print 'Rotating a node' + self.rotate(region=region) + + signal.signal(signal.SIGTERM, self.sighandler) + signal.signal(signal.SIGUSR1, self.sighandler) + + def pick_region(self): + """ Pick the region for the new node """ + + # Try and pick a region not present in the + # current list of nodes + regions = self.config.get_active_regions() + # Shuffle current regions + random.shuffle(self.config.region_ids) + + for reg in self.config.region_ids: + if reg not in regions: + return reg + + # All regions already present ? Pick a random one. + return random.choice(self.config.region_ids) + + def rotate(self, region=None): + """ Rotate the configuration to a new node """ + + proxy_out_label = None + # Pick the data-center + if region == None: + print 'Picking a region ...' + region = self.pick_region() + else: + print 'Using supplied region',region,'...' + + # Switch in the new linode from this region + new_proxy, proxy_id = self.make_new_instance(region) + + # Rotate another node + if self.config.policy == Policy.ROTATION_RANDOM: + proxy_out = self.config.get_proxy_for_rotation(use_random=True, input_region=region) + elif self.config.policy == Policy.ROTATION_NEW_REGION: + proxy_out = self.config.get_proxy_for_rotation(region_switch=True, input_region=region) + elif self.config.policy == Policy.ROTATION_LRU: + proxy_out = self.config.get_proxy_for_rotation(least_used=True, input_region=region) + elif self.config.policy == Policy.ROTATION_LRU_NEW_REGION: + proxy_out = self.config.get_proxy_for_rotation(least_used=True, region_switch=True, + input_region=region) + + # Switch in the new proxy + self.config.switch_in_proxy(new_proxy, proxy_id, region) + print 'Switched in new proxy',new_proxy + # Write configuration + self.config.write() + print 'Wrote new configuration.' + # Write new HAProxy LB template and reload ha proxy + ret1 = self.config.write_lb_config() + ret2 = self.config.reload_lb() + + if ret1 and ret2: + if proxy_out != None: + print 'Switched out proxy',proxy_out + proxy_out_id = int(self.config.get_proxy_id(proxy_out)) + + if proxy_out_id != 0: + proxy_out_label = self.get_instance_label(proxy_out_id) + print 'Removing switched out instance',proxy_out_id + self.delete_instance(proxy_out_id) + else: + 'Proxy id is 0, not removing proxy',proxy_out + else: + print 'Error - Did not switch out proxy as there was a problem in writing/restarting LB' + + if proxy_out_label != None: + # Get its label and assign it to the new linode + print 'Assigning label',proxy_out_label,'to new instance',proxy_id + time.sleep(5) + self.update_instance(proxy_id, + proxy_out_label, + self.config.group) + + # Post process the host + print 'Post-processing',new_proxy,'...' + self.post_process(new_proxy) + self.send_email(proxy_out, proxy_out_label, new_proxy, region) + + def post_process(self, ip): + """ Post-process a switched-in host """ + + # Sleep a bit before sshing + time.sleep(5) + cmd = post_process_cmd_template % (self.config.user, ip, iptables_restore_cmd) + print 'SSH command 1=>',cmd + os.system(cmd) + cmd = post_process_cmd_template % (self.config.user, ip, squid_restart_cmd) + print 'SSH command 2=>',cmd + os.system(cmd) + + def provision(self, count=8, add=False): + """ Provision an entirely fresh set of linodes after dropping current set """ + + if not add: + self.drop() + + num, idx = 0, 0 + + # If we are adding Linodes without dropping, start from current count + if add: + start = len(self.config.get_active_proxies()) + else: + start = 0 + + for i in range(start, start + count): + + # Do a round-robin on regions + region = self.config.region_ids[idx % len(self.config.region_ids) ] + try: + ip, lid = self.make_new_instance(region) + new_label = self.config.proxy_prefix + str(i+1) + self.update_instance(int(lid), + new_label, + self.config.group) + + num += 1 + except Exception, e: + print 'Error creating instance',e + + idx += 1 + + print 'Provisioned',num,' proxies.' + # Save latest proxy information + self.write_proxies() + + def write_proxies(self): + """ Write proxies to a file """ + + proxies_list = self.vps_command.get_proxies() + # Randomize it + for i in range(5): + random.shuffle(proxies_list) + + filename = self.config.proxylist + print >> open(filename, 'w'), '\n'.join(proxies_list) + print 'Saved current proxy configuration to {}'.format(filename) + + def test(self): + """ Function to be called in loop for testing """ + + proxy_out_label = '' + region = self.pick_region() + print 'Rotating proxy to new region',region,'...' + # Make a test IP + new_proxy, proxy_id = self.make_new_linode(region, test=True) + proxy_out = self.config.get_proxy_for_rotation(least_used=True, region_switch=True, + input_region=region) + + if proxy_out != None: + print 'Switched out proxy',proxy_out + proxy_out_id = int(self.config.get_proxy_id(proxy_out)) + proxy_out_label = self.linode_cmd.get_label(proxy_out_id) + + # Switch in the new proxy + self.config.switch_in_proxy(new_proxy, proxy_id, region) + print 'Switched in new proxy',new_proxy + # Write new HAProxy LB template and reload ha proxy + self.config.write_lb_config(test=True) + self.send_email(proxy_out, proxy_out_label, new_proxy, region) + + def stop(self): + """ Stop the rotator process """ + + try: + os.remove(self.hbf) + # Signal the event + self.alarm.set() + return True + except (IOError, OSError), e: + pass + + return False + + def sighandler(self, signum, stack): + """ Signal handler """ + + # This will be called when you want to stop the daemon + self.stop() + + def run(self): + """ Run as a background process, rotating proxies """ + + # Touch heartbeat file + open(self.hbf,'w').write('') + # Fork + print 'Daemonizing...' + daemonize('rotator.pid',logfile='rotator.log', drop=True) + print 'Proxy rotate daemon started.' + count = 1 + + while True: + # Wait on event object till woken up + self.alarm.wait(self.config.frequency) + status = self.alive() + if not status: + print 'Daemon signalled to exit. Quitting ...' + break + + print 'Rotating proxy node, round #%d ...' % count + if self.test_mode: + self.test() + else: + self.rotate() + count += 1 + + sys.exit(0) + + def create(self, region=3): + """ Create a new instance for testing """ + + print 'Creating new instance in region',region,'...' + new_proxy = self.make_new_instance(region, verbose=True) + + return new_proxy + + def send_email(self, proxy_out, label, proxy_in, region): + """ Send email upon switching of a proxy """ + + print 'Sending email...' + region = region_dict[region] + content = email_template % locals() + email_config = self.config.get_email_config() + + email_report.email_report(email_config, "%s", content) + + def alive(self): + """ Return whether I should be alive """ + + return os.path.isfile(self.hbf) + + def get_instance_label(self, instance_id): + """ Return instance label given instance id """ + pass + + def update_instance(self, instance_id, label, group=None): + """ Update the meta-data for the instance """ + pass + + def delete_instance(self, instance_id): + """ Delete a given instance given its id """ + pass + + def drop(self): + """ Drop all instances in current configuration (except the LB) """ + pass + From a75060b8b0917a1dd62f75826d3d2a3c913513ee Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:27:48 +0530 Subject: [PATCH 04/18] Major Refactoring --- rotate_proxies.py | 701 +++------------------------------------------- 1 file changed, 43 insertions(+), 658 deletions(-) diff --git a/rotate_proxies.py b/rotate_proxies.py index 2a8c46d..4cfb8ad 100644 --- a/rotate_proxies.py +++ b/rotate_proxies.py @@ -6,659 +6,8 @@ import argparse import os import sys -import time -import random -import collections -import operator -import uuid -import threading -import signal -import json -import email_report -from utils import daemonize, randpass, enum, LinodeCommand, AWSCommand -from botocore.exceptions import ClientError - -# Rotation Policies -Policy = enum('ROTATION_RANDOM', - # Least recently used - 'ROTATION_LRU', - # Switch to another region - 'ROTATION_NEW_REGION', - # LRU + New region - 'ROTATION_LRU_NEW_REGION') - -region_dict = {2: 'Dallas', - 3: 'Fremont', - 4: 'Atlanta', - 6: 'Newark', - 7: 'London', - 8: 'Tokyo', - 9: 'Singapore', - 10: 'Frankfurt'} - - -email_template = """ - -I just switched a proxy node in the proxy infrastructure. Details are below. - -In: %(label)s, %(proxy_in)s -Out: %(label)s, %(proxy_out)s - -Region: %(region)s - --- Linode proxy daemon - -""" - -# Post process command -post_process_cmd_template = """ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@%s "%s" """ -iptables_restore_cmd = "sudo iptables-restore < /etc/iptables.rules" -squid_restart_cmd = "sudo squid3 -f /etc/squid3/squid.conf" - -class ProxyConfig(object): - """ Class representing configuration of crawler proxy infrastructure """ - - def __init__(self, cfg='proxy.conf'): - """ Initialize proxy config from the config file """ - - self.parse_config(cfg) - # This is a file with each line of the form - # IPV4 address, datacenter code, linode-id, switch_in timestamp, switch_out timestamp - # E.g: 45.79.91.191, 3, 1446731065, 144673390 - try: - proxies = map(lambda x: x.strip().split(','), open(self.proxylist).readlines()) - # Proxy IP to (switch_in, switch_out) timestamp mappings - self.proxy_dict = {} - # Proxy IP to enabled mapping - self.proxy_state = {} - self.process_proxies(proxies) - except (OSError, IOError), e: - print e - sys.exit("Fatal error, proxy list input file " + self.proxylist + " not found!") - - try: - self.proxy_template = open(self.lb_template).read() - except (OSError, IOError), e: - print e - sys.exit("Fatal error, template config input file " + template_file + " not found!") - - def parse_config(self, cfg): - """ Parse the configuration file and load config """ - - self.config = json.load(open(cfg)) - for key,value in self.config.items(): - # Set attribute locally - setattr(self, key, value) - - # Do some further processing - self.frequency = float(self.frequency)*3600.0 - self.policy = eval('Policy.' + self.policy) - - def get_proxy_ips(self): - """ Return all proxy IP addresses as a list """ - - return self.proxy_state.keys() - - def get_active_proxies(self): - """ Return a list of all active proxies as a list """ - - return map(self.proxy_dict.get, filter(self.proxy_state.get, self.proxy_state.keys())) - - def process_proxies(self, proxies): - """ Process the proxy information to create internal dictionaries """ - - # Prepare the proxy region dict - for proxy_ip, region, proxy_id, switch_in, switch_out in proxies: - # If switch_in ==0: put current time - if int(float(switch_in))==0: - switch_in = int(time.time()) - if int(float(switch_out))==0: - switch_out = int(time.time()) - - if self.vps_provider == 'linode': - self.proxy_dict[proxy_ip] = [proxy_ip, int(region), int(proxy_id), int(float(switch_in)), int(float(switch_out))] - elif self.vps_provider == 'aws': - self.proxy_dict[proxy_ip] = [proxy_ip, int(region), proxy_id, int(float(switch_in)), int(float(switch_out))] - self.proxy_state[proxy_ip] = True - - print 'Processed',len(self.proxy_state),'proxies.' - - def get_proxy_for_rotation(self, - use_random=False, - least_used=False, - region_switch=False, - input_region=3): - """ Return a proxy IP address for rotation using the given settings. The - returned proxy will be replaced with a new proxy. - - @use_random - Means returns a random proxy from the current active list - @least_used - Returns a proxy IP which is the oldest switched out one - so we keep the switching more or less democratic. - @region_switch - Returns a proxy which belongs to a different region - from the new proxy. - @input_region - The region of the new proxy node - defaults to Fremont, CA. - - Note that if use_random is set to true, the other parameters are ignored. - - """ - - active_proxies = self.get_active_proxies() - print 'Active proxies =>',active_proxies - - if use_random: - # Pick a random proxy IP - proxy = random.choice(active_proxies) - print 'Returning proxy =>',proxy - proxy_ip = proxy[0] - - # Remove it from every data structure - self.switch_out_proxy(proxy_ip) - return proxy - - if least_used: - # Pick the oldest switched out proxy i.e one - # with smallest switched out value - proxies_used = sorted(active_proxies, - key=operator.itemgetter(-1)) - - print 'Proxies used =>',proxies_used - - if region_switch: - # Find the one with a different region from input - for proxy, reg, pi, si, so in proxies_used: - if reg != input_region: - print 'Returning proxy',proxy,'from region',reg - self.switch_out_proxy(proxy) - return proxy - - # If all regions are already in use, pick the last used - # proxy anyway - return proxies_used[0][0] - - if region_switch: - # Pick a random proxy not in the input region - proxies = active_proxies - random.shuffle(proxies) - - for proxy, reg, pi, si, so in proxies: - if reg != input_region: - print 'Returning proxy',proxy,'from region',reg - self.switch_out_proxy(proxy) - return proxy - - def __getattr__(self, name): - """ Return from local, else written from config """ - - try: - return self.__dict__[name] - except KeyError: - return self.config.get(name) - - def switch_out_proxy(self, proxy): - """ Switch out a given proxy IP """ - - # Disable it - self.proxy_state[proxy] = False - # Mark its switched out timestamp - self.proxy_dict[proxy][-1] = int(time.time()) - - def switch_in_proxy(self, proxy, proxy_id, region): - """ Switch in a given proxy IP """ - - # Mark its switched out timestamp - if self.vps_provider == 'linode': - self.proxy_dict[proxy] = [proxy, int(region), int(proxy_id), int(time.time()), int(time.time())] - elif self.vps_provider == 'aws': - self.proxy_dict[proxy] = [proxy, int(region), proxy_id, int(time.time()), int(time.time())] - # Enable it - self.proxy_state[proxy] = True - - def get_active_regions(self): - """ Return unique regions for which proxies are active """ - - regions = set() - for proxy,region,pi,si,so in self.proxy_dict.values(): - if self.proxy_state[proxy]: - regions.add(region) - - return list(regions) - - def write(self, disabled=False): - """ Write current state to an output file """ - - lines = [] - for proxy, reg, pi, si, so in self.proxy_dict.values(): - if disabled or self.proxy_state[proxy]: - lines.append('%s,%s,%s,%s,%s\n' % (proxy, str(reg), str(pi), str(int(si)), str(int(so)))) - - open(self.proxylist,'w').writelines(lines) - - def write_lb_config(self, disabled=False, test=False): - """ Write current proxy configuration into the load balancer config """ - - lines, idx = [], 1 - # Shuffle - items = self.proxy_dict.values() - for i in range(10): - random.shuffle(items) - - for proxy, reg, pi, si, so in items: - if self.proxy_state[proxy]: - lines.append('\tserver squid%d %s:8321 check inter 10000 rise 2 fall 5' % (idx, proxy)) - idx += 1 - - squid_config = "\n".join(lines) - content = self.proxy_template % locals() - # Write to temp file - tmpfile = '/tmp/.haproxy.cfg' - open(tmpfile,'w').write(content) - - # If running in test mode, don't do this! - if not test: - # Run as sudo - cmd = 'sudo cp %s %s; rm -f %s' % (tmpfile, self.lb_config, tmpfile) - os.system(cmd) - - self.reload_lb() - return True - - def reload_lb(self): - """ Reload the HAProxy load balancer """ - - return (os.system(self.lb_restart) == 0) - - def get_proxy_id(self, proxy): - """ Given proxy return its id """ - - return self.proxy_dict[proxy][2] - - def get_email_config(self): - """ Return email configuration """ - - return self.config['email'] - -class ProxyRotator(object): - """ Proxy rotation, provisioning & re-configuration with linode nodes """ - - def __init__(self, cfg='proxy.conf', test_mode=False, rotate=False, region=None): - self.config = ProxyConfig(cfg=cfg) - print 'Frequency set to',self.config.frequency,'seconds.' - # Test mode ? - self.test_mode = test_mode - # Event object - self.alarm = threading.Event() - # Clear the event - self.alarm.clear() - # Heartbeat file - self.hbf = '.heartbeat' - # Linode creation class - self.linode_cmd = LinodeCommand(verbose=True, config=self.config) - #AWS resource manager - self.aws_command = AWSCommand(config=self.config) - # If rotate is set, rotate before going to sleep - if rotate: - print 'Rotating a node' - self.rotate(region=region) - - signal.signal(signal.SIGTERM, self.sighandler) - signal.signal(signal.SIGUSR1, self.sighandler) - - def pick_region(self): - """ Pick the region for the new node """ - - # Try and pick a region not present in the - # current list of nodes - regions = self.config.get_active_regions() - # Shuffle current regions - random.shuffle(self.config.region_ids) - - for reg in self.config.region_ids: - if reg not in regions: - return reg - - # All regions already present ? Pick a random one. - return random.choice(self.config.region_ids) - - def make_new_linode(self, region, test=False, verbose=False): - """ Make a new linode in the given region """ - - # If calling as test, make up an ip - if test: - return '.'.join(map(lambda x: str(random.randrange(20, 100)), range(4))), random.randrange(10000, - 50000) - - tup = (region, - self.config.plan_id, - self.config.os_id, - self.config.image_id, - 'proxy_disk', - randpass()) - - print 'Making new linode in region',region,'...' - data = self.linode_cmd.linode_create(*tup) - - # data = os.popen(cmd).read() - if verbose: - print data - # The IP is the last line of the command - ip = data.strip().split('\n')[-1].strip().split()[-1].strip() - # Proxy ID - pid = data.strip().split('\n')[-3].strip().split()[-1].strip() - print 'I.P address of new linode is',ip - print 'ID of new linode is',pid - # Post process the host - print 'Post-processing',ip,'...' - self.post_process(ip) - return ip, pid - - def rotate(self, region=None): - """ Rotate the configuration to a new node """ - - proxy_out_label = None - # Pick the data-center - if region == None: - print 'Picking a region ...' - region = self.pick_region() - else: - print 'Using supplied region',region,'...' - - if self.config.vps_provider == 'linode': - # Switch in the new linode from this region - new_proxy, proxy_id = self.make_new_linode(region) - - elif self.config.vps_provider == 'aws': - # Switch in the new aws instance - new_proxy, proxy_id = self.make_new_ec2() - - # Rotate another node - if self.config.policy == Policy.ROTATION_RANDOM: - proxy_out = self.config.get_proxy_for_rotation(use_random=True, input_region=region) - elif self.config.policy == Policy.ROTATION_NEW_REGION: - proxy_out = self.config.get_proxy_for_rotation(region_switch=True, input_region=region) - elif self.config.policy == Policy.ROTATION_LRU: - proxy_out = self.config.get_proxy_for_rotation(least_used=True, input_region=region) - elif self.config.policy == Policy.ROTATION_LRU_NEW_REGION: - proxy_out = self.config.get_proxy_for_rotation(least_used=True, region_switch=True, - input_region=region) - - # Switch in the new proxy - self.config.switch_in_proxy(new_proxy, proxy_id, region) - print 'Switched in new proxy',new_proxy - # Write configuration - self.config.write() - print 'Wrote new configuration.' - # Write new HAProxy LB template and reload ha proxy - ret1 = self.config.write_lb_config() - ret2 = self.config.reload_lb() - if ret1 and ret2: - if proxy_out != None: - print 'Switched out proxy',proxy_out - proxy_out_id = int(self.config.get_proxy_id(proxy_out)) - if proxy_out_id != 0: - if self.config.vps_provider == 'linode': - proxy_out_label = self.linode_cmd.get_label(proxy_out_id) - print 'Removing switched out linode',proxy_out_id - self.linode_cmd.linode_delete(proxy_out_id) - elif self.config.vps_provider == 'aws': - print 'Removing switched out aws instance',proxy_out_id - self.aws_command.delete_ec2(proxy_out_id) - else: - 'Proxy id is 0, not removing proxy',proxy_out - else: - print 'Error - Did not switch out proxy as there was a problem in writing/restarting LB' - - if self.config.vps_provider == 'linode': - if proxy_out_label != None: - # Get its label and assign it to the new linode - print 'Assigning label',proxy_out_label,'to new linode',proxy_id - time.sleep(5) - self.linode_cmd.linode_update(int(proxy_id), - proxy_out_label, - self.config.group) - - # Post process the host - print 'Post-processing',new_proxy,'...' - self.post_process(new_proxy) - self.send_email(proxy_out, proxy_out_label, new_proxy, region) - - def send_email(self, proxy_out, label, proxy_in, region): - """ Send email upon switching of a proxy """ - - print 'Sending email...' - region = region_dict[region] - content = email_template % locals() - email_config = self.config.get_email_config() - - email_report.email_report(email_config, "%s", content) - - def post_process(self, ip): - """ Post-process a switched-in host """ - - # Sleep a bit before sshing - time.sleep(5) - cmd = post_process_cmd_template % (self.config.user, ip, iptables_restore_cmd) - print 'SSH command 1=>',cmd - os.system(cmd) - cmd = post_process_cmd_template % (self.config.user, ip, squid_restart_cmd) - print 'SSH command 2=>',cmd - os.system(cmd) - - def alive(self): - """ Return whether I should be alive """ - - return os.path.isfile(self.hbf) - - def create(self, region=3): - """ Create a new linode for testing """ - if self.config.vps_provider == 'linode': - print 'Creating new linode in region',region,'...' - new_proxy = self.make_new_linode(region, verbose=True) - elif self.config.vps_provider == 'aws': - print 'Creating new ec2 instance','...' - new_proxy = self.make_new_ec2() - - def drop(self): - """ Drop all the proxies in current configuration (except the LB) """ - - if self.config.vps_provider == 'linode': - print 'Dropping all proxies ...' - proxies = rotator.linode_cmd.linode_list_proxies() - for item in proxies.split('\n'): - if item.strip() == "": continue - ip,dc,lid,si,so = item.split(',') - print '\tDropping linode',lid,'with IP',ip,'from dc',dc,'...' - self.linode_cmd.linode_delete(int(lid)) - - elif self.config.vps_provider == 'aws': - print 'Dropping all proxies ...' - proxies = rotator.aws_command.list_proxies() - for item in proxies: - ip,_,instance_id = item.split(',') - print '\tDropping ec2',instance_id,'with IP',ip,'...' - self.aws_command.delete_ec2(instance_id) - print 'done.' - - def provision(self, count=8, add=False): - """ Provision an entirely fresh set of linodes after dropping current set """ - - if not add: - self.drop() - - num, idx = 0, 0 - - # If we are adding Linodes without dropping, start from current count - if add: - start = len(self.config.get_active_proxies()) - else: - start = 0 - - for i in range(start, start + count): - - if self.config.vps_provider == 'linode': - # region = self.pick_region() - # Do a round-robin on regions - region = self.config.region_ids[idx % len(self.config.region_ids) ] - try: - ip, lid = self.make_new_linode(region) - self.linode_cmd.linode_update(int(lid), - self.config.proxy_prefix + str(i+1), - self.config.group) - num += 1 - except Exception, e: - print 'Error creating linode',e - - elif self.config.vps_provider == 'aws': - try: - ip, instance_id = self.make_new_ec2() - num += 1 - except ClientError as e: - print 'Error creating aws ec2 instance ',e - - idx += 1 - - print 'Provisioned',num,' proxies.' - if self.config.vps_provider == 'linode': - proxies_list = rotator.linode_cmd.linode_list_proxies().strip().split('\n') - elif self.config.vps_provider == 'aws': - proxies_list = rotator.aws_command.list_proxies() - # Randomize it - for i in range(5): - random.shuffle(proxies_list) - - print >> open('proxies.list', 'w'), '\n'.join(proxies_list) - print 'Saved current proxy configuration to proxies.list' - - def test(self): - """ Function to be called in loop for testing """ - - proxy_out_label = '' - region = self.pick_region() - print 'Rotating proxy to new region',region,'...' - # Make a test IP - new_proxy, proxy_id = self.make_new_linode(region, test=True) - proxy_out = self.config.get_proxy_for_rotation(least_used=True, region_switch=True, - input_region=region) - - if proxy_out != None: - print 'Switched out proxy',proxy_out - proxy_out_id = int(self.config.get_proxy_id(proxy_out)) - proxy_out_label = self.linode_cmd.get_label(proxy_out_id) - - # Switch in the new proxy - self.config.switch_in_proxy(new_proxy, proxy_id, region) - print 'Switched in new proxy',new_proxy - # Write new HAProxy LB template and reload ha proxy - self.config.write_lb_config(test=True) - self.send_email(proxy_out, proxy_out_label, new_proxy, region) - - def stop(self): - """ Stop the rotator process """ - - try: - os.remove(self.hbf) - # Signal the event - self.alarm.set() - return True - except (IOError, OSError), e: - pass - - return False - - def sighandler(self, signum, stack): - """ Signal handler """ - - # This will be called when you want to stop the daemon - self.stop() - - def run(self, daemon=True): - """ Run as a background process, rotating proxies """ - - # Touch heartbeat file - open(self.hbf,'w').write('') - # Fork - if daemon: - print 'Daemonizing...' - daemonize('rotator.pid',logfile='rotator.log', drop=True) - else: - # Write PID anyway - open('rotator.pid','w').write(str(os.getpid())) - - print 'Proxy rotate daemon started.' - count = 1 - - while True: - # Wait on event object till woken up - self.alarm.wait(self.config.frequency) - status = self.alive() - if not status: - print 'Daemon signalled to exit. Quitting ...' - break - - print 'Rotating proxy node, round #%d ...' % count - if self.test_mode: - self.test() - else: - self.rotate() - count += 1 - - sys.exit(0) - # AWS code - def make_new_ec2(self, test=False, verbose=False): - # If calling as test, make up an ip - if test: - return '.'.join(map(lambda x: str(random.randrange(20, 100)), range(4))), random.randrange(10000, - 50000) - params = dict(ImageId=self.config.aws_image_id, - InstanceType=self.config.aws_instance_type, - KeyName=self.config.aws_key_name, - SecurityGroupIds=self.config.aws_security_groups, - SubnetId=self.config.aws_subnet_id , - DryRun=True) - - print 'Making new ec2...' - ec2_instance = self.aws_command.create_ec2(**params) - ec2_instance.wait_until_running() - time.sleep(10) - - ip = ec2_instance.public_ip_address - pid = ec2_instance.id - - # Post process the host - print 'Post-processing',ip,'...' - self.post_process(ip) - - return ip, pid - -if __name__ == "__main__": - parser = argparse.ArgumentParser(prog='rotate_proxies') - parser.add_argument('-C','--conf',help='Use the given configuration file', default='proxy.conf') - parser.add_argument('-s','--stop',help='Stop the currently running daemon', action='store_true') - parser.add_argument('-t','--test',help='Run the test function to test the daemon', action='store_true') - parser.add_argument('-n','--nodaemon',help='Run in foreground', action='store_true',default=False) - parser.add_argument('-c','--create',help='Create a proxy linode', action='store_true',default=False) - parser.add_argument('-r','--region',help='Specify a region when creating a linode', default=3, type=int) - parser.add_argument('-R','--rotate',help='Rotate a node immediately and go to sleep', default=False, - action='store_true') - parser.add_argument('-D','--drop',help='Drop the current configuration of proxies (except LB)', - default=False,action='store_true') - parser.add_argument('-P','--provision',help='Provision a fresh set of proxy linodes',default=False, - action='store_true') - parser.add_argument('-A','--add',help='Add a new set of linodes to existing set',default=False, - action='store_true') - parser.add_argument('-N','--num',help='Number of new linodes to provision or add (use with -P or -A)',type=int, - default=8) - - parser.add_argument('-w','--writeconfig',help='Load current Linode proxies configuration and write a fresh proxies.list config file', action='store_true') - parser.add_argument('-W','--writelbconfig',help='Load current Linode proxies configuration and write a fresh HAProxy config to /etc/haproxy/haproxy.cfg', action='store_true') - parser.add_argument('--restart',help='Restart the daemon',action='store_true') - - args = parser.parse_args() - # print args - - rotator = ProxyRotator(cfg=args.conf, - test_mode = args.test, - rotate=args.rotate) +def process_args(rotator, args): if args.test: print 'Testing the daemon' @@ -687,10 +36,7 @@ def make_new_ec2(self, test=False, verbose=False): if args.writeconfig: # Load current proxies config and write proxies.list file - if rotator.config.vps_provider == 'linode': - print >> open('proxies.list', 'w'), rotator.linode_cmd.linode_list_proxies().strip() - elif rotator.config.vps_provider == 'aws': - print >> open('proxies.list', 'w'), '\n'.join(rotator.aws_command.list_proxies()) + rotator.write_proxies() print 'Saved current proxy configuration to proxies.list' sys.exit(0) @@ -717,7 +63,46 @@ def make_new_ec2(self, test=False, verbose=False): print 'Starting...' os.system('python rotate_proxies.py') - sys.exit(1) + sys.exit(1) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog='rotate_proxies') + parser.add_argument('-C','--conf',help='Use the given configuration file', default='proxy.conf') + parser.add_argument('-s','--stop',help='Stop the currently running daemon', action='store_true') + parser.add_argument('-t','--test',help='Run the test function to test the daemon', action='store_true') + parser.add_argument('-c','--create',help='Create a proxy linode', action='store_true',default=False) + parser.add_argument('-r','--region',help='Specify a region when creating a linode', default=3, type=int) + parser.add_argument('-R','--rotate',help='Rotate a node immediately and go to sleep', default=False, + action='store_true') + parser.add_argument('-D','--drop',help='Drop the current configuration of proxies (except LB)', + default=False,action='store_true') + parser.add_argument('-P','--provision',help='Provision a fresh set of proxy linodes',default=False, + action='store_true') + parser.add_argument('-A','--add',help='Add a new set of linodes to existing set',default=False, + action='store_true') + parser.add_argument('-N','--num',help='Number of new linodes to provision or add (use with -P or -A)',type=int, + default=10) + + parser.add_argument('-w','--writeconfig',help='Load current Linode proxies configuration and write a fresh proxies.list config file', action='store_true') + parser.add_argument('-W','--writelbconfig',help='Load current Linode proxies configuration and write a fresh HAProxy config to /etc/haproxy/haproxy.cfg', action='store_true') + parser.add_argument('--restart',help='Restart the daemon',action='store_true') + parser.add_argument('-T','--target',help='Target VPS platform (linode, aws)',default='linode') - rotator.run(daemon=(not args.nodaemon)) + args = parser.parse_args() + # print args + + if args.target == 'linode': + linode = __import__('linode') + rotator = linode.LinodeProxyRotator(cfg=args.conf, + test_mode = args.test, + rotate=args.rotate) + else: + aws = __import__('aws') + rotator = aws.AwsProxyRotator(cfg=args.conf, + test_mode = args.test, + rotate=args.rotate) + + + process_args(rotator, args) + rotator.run() From 6ce5e44d6b78b5c811eeebb9e9771d970c5790d4 Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:27:55 +0530 Subject: [PATCH 05/18] Major Refactoring --- aws.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 aws.py diff --git a/aws.py b/aws.py new file mode 100644 index 0000000..ab0ae5a --- /dev/null +++ b/aws.py @@ -0,0 +1,80 @@ +import random +import boto3 +from base import ProxyRotator + +class AWSCommand(object): + '''Class encapsulating the aws ec2 API''' + + def __init__(self, config=None): + self.ec2 = boto3.resource('ec2') + self.config = config + + def create_ec2(self, **params): + return self.ec2.create_instances(MaxCount=1, MinCount=1, **params)[0] + + def get_proxies(self): + proxies = [] + filters=[ + {'Name':'image-id', 'Values':[self.config.aws_image_id]}, + {'Name': 'instance-state-name', 'Values': ['running']} + ] + for instance in self.ec2.instances.filter(Filters=filters): + proxies.append(','.join([instance.public_ip_address, '0', instance.id,'0','0'])) + return proxies + + def delete_ec2(self, instance_id): + instance = self.ec2.Instance(instance_id) + instance.terminate() + instance.wait_until_terminated() + + +class AwsProxyRotator(ProxyRotator): + """ AWS implementation of ProxyRotator """ + + def __init__(self, cfg='proxy.conf', test_mode=False, rotate=False, region=None): + super(AwsProxyRotator, self).__init__(cfg, test_mode, rotate, region) + #AWS resource manager + self.aws_command = AWSCommand(config=self.config) + self.vps_command = self.aws_command + + def delete_instance(self, instance_id): + """ Delete instance by id """ + return self.aws_command.delete_ec2(instance_id) + + def make_new_instance(self, region=None, test=False, verbose=False): + # If calling as test, make up an ip + if test: + return '.'.join(map(lambda x: str(random.randrange(20, 100)), range(4))), random.randrange(10000, + 50000) + params = dict(ImageId=self.config.aws_image_id, + InstanceType=self.config.aws_instance_type, + KeyName=self.config.aws_key_name, + SecurityGroupIds=self.config.aws_security_groups, + SubnetId=self.config.aws_subnet_id , + DryRun=True) + + print 'Making new ec2...' + ec2_instance = self.aws_command.create_ec2(**params) + ec2_instance.wait_until_running() + time.sleep(10) + + ip = ec2_instance.public_ip_address + pid = ec2_instance.id + + # Post process the host + print 'Post-processing',ip,'...' + self.post_process(ip) + + return ip, pid + + def drop(self): + """ Drop all instances in current configuration (except the LB) """ + + print 'Dropping all proxies ...' + proxies = self.aws_command.get_proxies() + + for item in proxies: + ip,_,instance_id = item.split(',') + print '\tDropping ec2',instance_id,'with IP',ip,'...' + self.aws_command.delete_ec2(instance_id) + From 017c9098c3a56ccf416e3a6d9c9624d0f4fcd59a Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:28:00 +0530 Subject: [PATCH 06/18] Major Refactoring --- utils.py | 70 +------------------------------------------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/utils.py b/utils.py index e3f2481..ad98172 100644 --- a/utils.py +++ b/utils.py @@ -5,7 +5,6 @@ import pwd import uuid import functools -import boto3 def enum(*sequential, **named): enums = dict(zip(sequential, range(len(sequential))), **named) @@ -95,74 +94,7 @@ def daemonize(pidfile, logfile=None, user='ubuntu', drop=True): sys.stdin.close() sys.stdout=sys.stderr=log -class LinodeCommand(object): - """ Class encapsulating linode CLI commands """ - - def __init__(self, binary='linode', verbose=False, config=None): - self.binary = binary - self.verbose = verbose - self.cmd_template = {'create': 'create -d %d -p %d -o %d -i %d -l %s -r %s', - 'delete': 'delete -l %d', - 'list_proxies': 'find -g %s -s %s' % (config.group, config.proxylb), - 'info': 'info -l %d', - 'update': 'update -l %d -L %s -g %s' - } - # Dynamically create command methods - self.dyn_create() - - def _run(self, command, *args): - """ Run a command and return the output """ - - template = self.cmd_template.get(command) - if template == None: - print 'No such command configured =>',command - return -1 - - cmd = ' '.join((self.binary, template % args)) - if self.verbose: print 'Command is',cmd - return os.popen(cmd).read() - - def dyn_create(self): - """ Dynamically create linode methods """ - - for cmd in self.cmd_template: - method_name = 'linode_' + cmd - method = functools.partial(self._run, cmd) - if self.verbose: print 'Dyn-creating method',method_name,'...' - setattr(self, method_name, method) - - def get_label(self, linode_id): - """ Return the label, given the linode id """ - - data = self.linode_info(linode_id) - return data.split('\n')[0].split(':')[-1].strip() - -class AWSCommand(object): - '''Class encapsulating the aws ec2 API''' - def __init__(self, config=None): - self.ec2 = boto3.resource('ec2') - self.config = config - - def create_ec2(self, **params): - return self.ec2.create_instances(MaxCount=1, MinCount=1, **params)[0] - - def list_proxies(self): - proxies = [] - filters=[ - {'Name':'image-id', 'Values':[self.config.aws_image_id]}, - {'Name': 'instance-state-name', 'Values': ['running']} - ] - for instance in self.ec2.instances.filter(Filters=filters): - proxies.append(','.join([instance.public_ip_address, '0', instance.id,'0','0'])) - return proxies - - def delete_ec2(self, instance_id): - instance = self.ec2.Instance(instance_id) - instance.terminate() - instance.wait_until_terminated() - if __name__ == "__main__": - l = LinodeCommand() - l.get_label(int(sys.argv[1])) + pass From d8fbc28e9b9562aae4a41c9fa0a3ec35dc7c7a3f Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:28:05 +0530 Subject: [PATCH 07/18] Major Refactoring --- email_report.py | 1 - 1 file changed, 1 deletion(-) diff --git a/email_report.py b/email_report.py index 46552cb..1193239 100644 --- a/email_report.py +++ b/email_report.py @@ -3,7 +3,6 @@ import sys import socket import os -import ses_email import send_gmail def email_report(config, template, content): From 97ab761063fff36755c33e947a33a9b602e363f0 Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:28:19 +0530 Subject: [PATCH 08/18] Major Refactoring --- proxy.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxy.conf b/proxy.conf index d7c0c19..5a5a6fd 100644 --- a/proxy.conf +++ b/proxy.conf @@ -5,7 +5,7 @@ "policy": "ROTATION_LRU_NEW_REGION", "rotate": 1, "frequency": 72, - "image_id": 1121781, + "image_id": 2520746, "plan_id": 1, "os_id": 140, "region_ids": [2,3,4,6,7,9,10], @@ -24,6 +24,7 @@ "to_email": ["anandpillai@letterboxes.org"], "email_subject": "Linode proxy switch report: %s from %s" }, + "vps_provider":"aws", "aws_image_id":"ami-f104ec8c", "aws_instance_type":"t2.micro", From fda7be9fda3dc9a8e949b19c6405f79e1d6efaad Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:35:29 +0530 Subject: [PATCH 09/18] Return ip and pid from make_new_instance --- linode.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linode.py b/linode.py index f6f978c..f47a19c 100644 --- a/linode.py +++ b/linode.py @@ -99,6 +99,8 @@ def make_new_instance(self, region, test=False, verbose=False): print 'Post-processing',ip,'...' self.post_process(ip) + return ip, pid + def update_instance(self, instance_id, label, group=None): """ Update meta-data for a new instance """ From 043f50660e133fc2f6522decd1ff7eb8834a65dc Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:42:33 +0530 Subject: [PATCH 10/18] image id --- proxy.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy.conf b/proxy.conf index 5a5a6fd..b9bddc8 100644 --- a/proxy.conf +++ b/proxy.conf @@ -5,7 +5,7 @@ "policy": "ROTATION_LRU_NEW_REGION", "rotate": 1, "frequency": 72, - "image_id": 2520746, + "image_id": 0, "plan_id": 1, "os_id": 140, "region_ids": [2,3,4,6,7,9,10], From a49257c51f9e578a1a43872786c5b0dcaac6b67f Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:47:58 +0530 Subject: [PATCH 11/18] Added scripts --- scripts/get_linode_image_id.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 scripts/get_linode_image_id.py diff --git a/scripts/get_linode_image_id.py b/scripts/get_linode_image_id.py new file mode 100644 index 0000000..b357b8d --- /dev/null +++ b/scripts/get_linode_image_id.py @@ -0,0 +1,18 @@ +import json + +def query_linode_image_id(): + """ Query for linode image id and write to proxy.conf """ + + image_id = raw_input('Enter linode image id: ') + try: + im_id = int(image_id) + cfg = json.load(open('proxy.conf')) + cfg['image_id'] = im_id + json.dump(cfg, open('proxy.conf','w'), indent=4) + except ValueError, e: + print 'Invalid image id=>',image_id + except Exception, e: + print 'Error updating proxy.conf =>',e + +if __name__ == "__main__": + query_linode_image_id() From 5623a6e491d62317ea9b4d04f96e092ed6d1957f Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Tue, 24 Sep 2019 17:51:38 +0530 Subject: [PATCH 12/18] Added requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8784310 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +https://github.com/anvetsu/pylinode/archive/master.zip From e7810897f8a8d68a8ddced7c93d3e6458c992448 Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Wed, 2 Oct 2019 12:23:49 +0530 Subject: [PATCH 13/18] Removing all hard-coded linode references --- base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/base.py b/base.py index 9f09fb8..97894d1 100644 --- a/base.py +++ b/base.py @@ -60,7 +60,7 @@ def rotate(self, region=None): else: print 'Using supplied region',region,'...' - # Switch in the new linode from this region + # Switch in the new proxy from this region new_proxy, proxy_id = self.make_new_instance(region) # Rotate another node @@ -99,7 +99,7 @@ def rotate(self, region=None): print 'Error - Did not switch out proxy as there was a problem in writing/restarting LB' if proxy_out_label != None: - # Get its label and assign it to the new linode + # Get its label and assign it to the new proxy print 'Assigning label',proxy_out_label,'to new instance',proxy_id time.sleep(5) self.update_instance(proxy_id, @@ -124,14 +124,14 @@ def post_process(self, ip): os.system(cmd) def provision(self, count=8, add=False): - """ Provision an entirely fresh set of linodes after dropping current set """ + """ Provision an entirely fresh set of proxies after dropping current set """ if not add: self.drop() num, idx = 0, 0 - # If we are adding Linodes without dropping, start from current count + # If we are adding without dropping, start from current count if add: start = len(self.config.get_active_proxies()) else: @@ -177,14 +177,14 @@ def test(self): region = self.pick_region() print 'Rotating proxy to new region',region,'...' # Make a test IP - new_proxy, proxy_id = self.make_new_linode(region, test=True) + new_proxy, proxy_id = self.make_new_instance(region, test=True) proxy_out = self.config.get_proxy_for_rotation(least_used=True, region_switch=True, input_region=region) if proxy_out != None: print 'Switched out proxy',proxy_out proxy_out_id = int(self.config.get_proxy_id(proxy_out)) - proxy_out_label = self.linode_cmd.get_label(proxy_out_id) + proxy_out_label = self.get_instance_label(proxy_out_id) # Switch in the new proxy self.config.switch_in_proxy(new_proxy, proxy_id, region) From e418f90c3724307ae79ec969f447bff2c799b936 Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Wed, 2 Oct 2019 12:23:54 +0530 Subject: [PATCH 14/18] Removing all hard-coded linode references --- rotate_proxies.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/rotate_proxies.py b/rotate_proxies.py index 4cfb8ad..86af943 100644 --- a/rotate_proxies.py +++ b/rotate_proxies.py @@ -1,11 +1,13 @@ """ -Script to auto-rotate and configure Linodes as squid proxies. +Script to auto-rotate and configure squid proxy farms using multiple +VPS backends using their APIs. """ import argparse import os import sys +import signal def process_args(rotator, args): @@ -15,17 +17,17 @@ def process_args(rotator, args): sys.exit(0) if args.add != 0: - print 'Adding new set of',args.num,'linode proxies ...' + print 'Adding new set of',args.num,'proxies ...' rotator.provision(count = int(args.num), add=True) sys.exit(0) if args.provision != 0: - print 'Provisioning fresh set of',args.num,'linode proxies ...' + print 'Provisioning fresh set of',args.num,'proxies ...' rotator.provision(count = int(args.num)) sys.exit(0) if args.create: - print 'Creating new linode...' + print 'Creating new instance...' rotator.create(int(args.region)) sys.exit(0) @@ -70,21 +72,21 @@ def process_args(rotator, args): parser.add_argument('-C','--conf',help='Use the given configuration file', default='proxy.conf') parser.add_argument('-s','--stop',help='Stop the currently running daemon', action='store_true') parser.add_argument('-t','--test',help='Run the test function to test the daemon', action='store_true') - parser.add_argument('-c','--create',help='Create a proxy linode', action='store_true',default=False) - parser.add_argument('-r','--region',help='Specify a region when creating a linode', default=3, type=int) + parser.add_argument('-c','--create',help='Create a proxy instance', action='store_true',default=False) + parser.add_argument('-r','--region',help='Specify a region when creating an instance', default=3, type=int) parser.add_argument('-R','--rotate',help='Rotate a node immediately and go to sleep', default=False, action='store_true') parser.add_argument('-D','--drop',help='Drop the current configuration of proxies (except LB)', default=False,action='store_true') - parser.add_argument('-P','--provision',help='Provision a fresh set of proxy linodes',default=False, + parser.add_argument('-P','--provision',help='Provision a fresh set of proxy instances',default=False, action='store_true') - parser.add_argument('-A','--add',help='Add a new set of linodes to existing set',default=False, + parser.add_argument('-A','--add',help='Add a new set of instances to existing farm',default=False, action='store_true') - parser.add_argument('-N','--num',help='Number of new linodes to provision or add (use with -P or -A)',type=int, + parser.add_argument('-N','--num',help='Number of new instances to provision or add (use with -P or -A)',type=int, default=10) - parser.add_argument('-w','--writeconfig',help='Load current Linode proxies configuration and write a fresh proxies.list config file', action='store_true') - parser.add_argument('-W','--writelbconfig',help='Load current Linode proxies configuration and write a fresh HAProxy config to /etc/haproxy/haproxy.cfg', action='store_true') + parser.add_argument('-w','--writeconfig',help='Load current proxies configuration and write a fresh proxies.list config file', action='store_true') + parser.add_argument('-W','--writelbconfig',help='Load current proxies configuration and write a fresh HAProxy config to /etc/haproxy/haproxy.cfg', action='store_true') parser.add_argument('--restart',help='Restart the daemon',action='store_true') parser.add_argument('-T','--target',help='Target VPS platform (linode, aws)',default='linode') @@ -96,7 +98,7 @@ def process_args(rotator, args): rotator = linode.LinodeProxyRotator(cfg=args.conf, test_mode = args.test, rotate=args.rotate) - else: + elif args.target == 'aws': aws = __import__('aws') rotator = aws.AwsProxyRotator(cfg=args.conf, test_mode = args.test, From 2358099795f94e07b5315a82c2be766d09da59de Mon Sep 17 00:00:00 2001 From: Anand B Pillai Date: Wed, 2 Oct 2019 12:24:30 +0530 Subject: [PATCH 15/18] Removing all hard-coded linode references --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 8291b7b..8b83b2e 100644 --- a/config.py +++ b/config.py @@ -36,7 +36,7 @@ Region: %(region)s --- Linode proxy daemon +-- Proxy Rotator Daemon """ @@ -54,7 +54,7 @@ def __init__(self, cfg='proxy.conf'): self.parse_config(cfg) # This is a file with each line of the form - # IPV4 address, datacenter code, linode-id, switch_in timestamp, switch_out timestamp + # IPV4 address, datacenter code, instance-id, switch_in timestamp, switch_out timestamp # E.g: 45.79.91.191, 3, 1446731065, 144673390 try: proxies = map(lambda x: x.strip().split(','), open(self.proxylist).readlines()) From 9268f2903d031eb31aaa092b2d1394e4524ea020 Mon Sep 17 00:00:00 2001 From: DeepSource Bot Date: Wed, 6 May 2020 18:35:51 +0000 Subject: [PATCH 16/18] Add .deepsource.toml --- .deepsource.toml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 0000000..25bc3d7 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,8 @@ +version = 1 + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" From 0d221d190100f36a03fd80583039fa5558940ef5 Mon Sep 17 00:00:00 2001 From: DeepSource Bot Date: Wed, 6 May 2020 18:36:11 +0000 Subject: [PATCH 17/18] Update .deepsource.toml From 9ca8323e2139442b401241efb743fa6a8406071a Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Wed, 6 May 2020 18:41:16 +0000 Subject: [PATCH 18/18] Remove unused imports --- send_gmail.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/send_gmail.py b/send_gmail.py index 5b06589..e4430a9 100644 --- a/send_gmail.py +++ b/send_gmail.py @@ -3,12 +3,7 @@ Send email via gmail SMTP """ - -import os -import sys -import optparse import smtplib -import time from email.MIMEMultipart import MIMEMultipart from email.MIMEText import MIMEText