plog95/plog95.py

#! /usr/bin/env python3

# Laurent GUERBY 20150728
# parse log-netdev-20120629.txt compute 95th centile 5mn
# python plog95.py b 0 eth0.2301  log-netdev-201506*.txt
import datetime
import sys
import re
import gzip
import lzma

def to_date(s):
    dt=datetime.datetime.strptime(s,"%a %b %d %H:%M:%S %Z %Y")
    return dt


b_or_p=sys.argv[1] # "b" or "p"
txrx=int(sys.argv[2]) # 0 for RX 1 for TX
iface95_l = sys.argv[3].split(':')

#=== Thu Jun 28 22:00:08 UTC 2012 ===
#Inter-|   Receive                                                |  Transmit
# face |bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
#    lo:2746879057 24561707    0    0    0     0          0         0 2746879057 24561707    0    0    0     0       0          0

# d[time][inter][byte/packet][rx/tx]

d={}
t_l=[]

for fname in sys.argv[4:]:
   f=None
   if fname.endswith(".xz"):
       f=lzma.open(fname, 'rt', encoding='utf-8')
   elif fname.endswith(".gz"):
       f=gzip.open(fname, 'rt', encoding='utf-8')
   else:
       f=open(fname, 'r', encoding='utf-8')
   line_i=0
   for l in f:
       line_i+=1
       # l = l.decode('ascii');
       if l.startswith("=== ") and l.endswith(" ===\n"):
           t_s=to_date(l[4:-5])
           d[t_s]={}
           t_l.append(t_s)
       elif l.startswith("Inter"):
           pass
       elif l.startswith(" face"):
           pass
       else:
           try:
               header,data=l[:-1].split(":")
           except:
               print(l)
               print(fname)
               print(line_i)
               #raise
           iface=header.strip(" ")
           if not iface in iface95_l: continue
           bytes=[int(x) for x in re.findall(r'\w+', data)]
           d[t_s][iface]={"b":[bytes[0],bytes[8]],"p":[bytes[1],bytes[9]]}

#print(len(t_l))

iface_d={}
for t in t_l:
    for i in d[t].keys():
        iface_d[i]=None

iface_l=iface_d.keys()

prev={}
t0=t_l[0]
for i in iface_l:
    prev[i]=d[t0][i][b_or_p][txrx]
tn0=t_l[0]
mb_l=[]
for t in t_l:
    # print(t, " ", tn0, " ", tn0 + datetime.timedelta(minutes=5))
    if (t < tn0 + datetime.timedelta(minutes=5)): continue
    delta = (t - tn0).seconds
    # if (delta > 450): print("xxxx ", tn0, " ", t, " ", delta)
    tn0=t
    tot_mbit_s = 0.0;
    for i in iface_l:
        y=0
        x=None
        if i in d[t]:
            x=d[t][i][b_or_p][txrx]
        if prev[i]!=None and x!=None:
            y=x-prev[i]
        if y<0.0: y=0
        mbit_s=float(y)*8.0/(delta*1e6)
        # if mbit_s>600.0: print(i, " ", t, " ", mbit_s)
        if mbit_s>10000.0: mbit_s=0.0
        tot_mbit_s += mbit_s;
         prev[i]=x
    mb_l.append(tot_mbit_s)

mb_l.sort()
mb_n=len(mb_l)
print("nb",mb_n,"avg",sum(mb_l)/mb_n,"median",mb_l[int(mb_n/2)],"90eme",mb_l[int(0.90*mb_n)],"95eme",mb_l[int(0.95*mb_n)],"max",mb_l[mb_n-1])