plog95/plog95.py

#! /usr/bin/env python3

# Laurent GUERBY 20150728
# parse log-netdev-20120629.txt compute 95th centile 5mn
# python plog95.py b 0 eth0.2301  log-netdev-201506*.txt
import datetime
import sys
import re
import gzip
import lzma

def to_date(s):
    dt=datetime.datetime.strptime(s,"%a %b %d %H:%M:%S %Z %Y")-datetime.datetime(1899,12,30)
    return dt.days+(dt.seconds+dt.microseconds*1.0e-6)/86400.0

def to_datetime(xd):
    return datetime.datetime(1900,1,1)+datetime.timedelta(xd)




b_or_p=sys.argv[1] # "b" or "p"
txrx=int(sys.argv[2]) # 0 for RX 1 for TX
iface95=sys.argv[3]

#=== Thu Jun 28 22:00:08 UTC 2012 ===
#Inter-|   Receive                                                |  Transmit
# face |bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
#    lo:2746879057 24561707    0    0    0     0          0         0 2746879057 24561707    0    0    0     0       0          0

# d[time][inter][byte/packet][rx/tx]

d={}
t_l=[]

for fname in sys.argv[4:]:
   f=None
   if fname.endswith(".xz"):
       f=lzma.open(fname)
   elif fname.endswith(".gz"):
       f=gzip.open(fname)
   else:
       f=open(fname)
   line_i=0
   for l in f:
       line_i+=1
       # l = l.decode('ascii');
       if l.startswith("=== ") and l.endswith(" ===\n"):
           t_s=to_date(l[4:-5])
           d[t_s]={}
           t_l.append(t_s)
       elif l.startswith("Inter"):
           pass
       elif l.startswith(" face"):
           pass
       else:
           try:
               header,data=l[:-1].split(":")
           except:
               print(l)
               print(fname)
               print(line_i)
               #raise
           iface=header.strip(" ")
           if iface!=iface95: continue
           bytes=[int(x) for x in re.findall(r'\w+', data)]
           d[t_s][iface]={"b":[bytes[0],bytes[8]],"p":[bytes[1],bytes[9]]}

#print(len(t_l))

iface_d={}
for t in t_l:
    for i in d[t].keys():
        iface_d[i]=None

iface_l=iface_d.keys()
#iface_l.sort()
#print(len(iface_l))

#print "T,",",".join(iface_l)
prev={}
for i in iface_l:
    prev[i]=None
t0=t_l[0]
tn0=0
mb_l=[]
for t in t_l:
    tn=int(86400.0*(t-t0)/300.0)
    if tn<=tn0 and t!=t0: continue
    tn0=tn
    s="%f"%t
    for i in iface_l:
        y=0
        x=None
        if i in d[t]:
            x=d[t][i][b_or_p][txrx]
        if prev[i]!=None and x!=None:
            y=x-prev[i]
        if y<0.0: y=0
        s+=",%d"%y
        mbit_s=float(y)*8.0/300.0e6
        #if mbit_s>600.0: print to_datetime(t),mbit_s
        if mbit_s>10000.0: mbit_s=0.0
        mb_l.append(mbit_s)
        prev[i]=x
    #print(s)

mb_l.sort()
mb_n=len(mb_l)
print("nb",mb_n,"avg",sum(mb_l)/mb_n,"median",mb_l[int(mb_n/2)],"90eme",mb_l[int(0.90*mb_n)],"95eme",mb_l[int(0.95*mb_n)],"max",mb_l[mb_n-1])