Python

import csv import re import os import argparse import json import time import datetime import requests import asyncio import logging import logging.handlers #from memory_profiler import profile #from pympler.tracker import SummaryTracker   def follow_file(fname, cnt):     """     Tail a given file and returns each line to the generator caller     """     cur_file = open(fname, "r")     cur_ino = os.fstat(cur_file.fileno()).st_ino     while True:         while True:             line = cur_file.readline()             cnt = cnt + 1             if not line:                 break             yield cnt, line         try:             if os.stat(fname).st_ino != cur_ino:                 new_file = open(fname, "r")                 cur_file.close()                 cur_file = new_file                 cur_ino = os.fstat(cur_file.fileno()).st_ino                 continue         except IOError:             logging.warning("{} IOError".format(fname))             pass         time.sleep(WAIT_FILE_SEC)   def process_line(line, idx):     """     Process one line at time to create a REST payload message     """     ev_dict = {}     line = line.replace('\"', '')     pat = r"(\w+=.*?),"     fields = re.findall(pat, line, flags=0)     for field in fields:         try:             key, value = field.split('=',1)             if (key in ev_dict):                 next             else:                 ev_dict[key] = value         except:             next     try:         ev_host = ev_dict['entity'].split('-')[-1]         ev_event = ev_dict['event']         ev_severity = ev_dict['eventSeverity']         ev_time = ev_dict['time']         ev_time = ev_time.replace(']', '')         ev_type = ev_dict['eventType']         ev_description = ev_dict['description']         ev_id = "{}-{}".format(line.split(' ')[0], idx)         ev_entity = ev_dict['entity']         ev_pub = ev_dict['publisher']         ev_state = ev_dict['state']     except:         logging.warning(line)         return None     payload = {         'app_key': APP_KEY,         'status': ev_severity,         'host': ev_host,         'timestamp': ev_time,         'check' : ev_event,         'description': ev_description,         'my_unique_attribute': ev_id,         'entity' : ev_entity,         'publisher' : ev_pub,         'state' : ev_state,         'supportcard' : ev_supportcard,         'wiki' : ev_wiki     }     return payload   #@profile def send_payload(payload, idx):     """     Send a single data payload to via REST API     """     url = "https://bpcom/nom/dev/events"     #ret_msg = {}     st = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")     response = requests.post(url, headers=HEADERS, data=json.dumps(payload))     et = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")     logging.info("{} {} {} {} {}".format(idx, payload['my_unique_attribute'], response.status_code, st, et))     #return ret_msg   if __name__  == '__main__':     parser = argparse.ArgumentParser(description= "feeder")     parser.add_argument("--test", "-t", action='store_true', help='test')     parser.add_argument("--infile", "-i", type=str, help="manual input file")     parser.add_argument("--start", "-s", type=int, help="start sending message at #")     parser.add_argument("--feed", "-f", type=str, help='FEED: \n")      args = parser.parse_args()     if (args.infile is not None):         amber_file = args.infile         logfile = "amber_panda.log"     else:         amber_feed = args.feed         if (amber_feed is None):             amber_feed = 'sentinel'         amber_file = AMBERFILES[amber_feed]         logfile = LOGFILES[amber_feed]     if (args.start is not None):         start_msg_idx = args.start     else:         start_msg_idx = 0     logging.basicConfig(filename=logfile, level=logging.INFO,                         format='%(asctime)s %(levelname)-8s %(message)s',                         datefmt ='%Y-%m-%d %H:%M:%S')     logging.info("Starts")     i = 0     loop = asyncio.get_event_loop()     for i, ev_line in follow_file(amber_file, i):         payload = process_line(ev_line, i)         if (payload is not None and i >= start_msg_idx):             future = loop.run_in_executor(None, send_payload, payload, i)             logging.info("Parsing AMBER Message {}".format(i))         else:             if (payload is None):                 logging.warning("{} Invalid Payload".format(i))             else:                 if (i % 100 == 0):                     logging.info("Skip {}... Messages".format(i))         if (args.test):             if (i > TEST_MSG_CNT):                 break         del payload     time_end_str = datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")     logging.info("End At " + time_end_str)