pcap_PE.py

import numpy as np
import pandas as pd
import pcap
import struct
import scipy.stats
import os
import re
# identify inbound and outbound traffic by site ip
# interval: in bytes
# convert string into integer for unique count
# flow: just check 4-tuples, i.e., srcIP, destIP, sport, dport
# 10Gb link, 20ms ~ 25k pkts ~ 25MB 

# identify incoming and outgoing traffic
campusNet_mask = '131.225.255.255' 
#campusNet_mask = '192.168.255.255'   # smallFlows.pcap, CAID Anomaly Detection
#campusNet_mask = '172.16.255.255'   # test.pcap, bigFlows.pcap
MAX_Ratio = 1e6
def hex_conv(hexs):
    return (int(hexs[0:2], 16)<<40) | (int(hexs[3:5], 16)<<32) | (int(hexs[6:8], 16)<24) | (int(hexs[9:11], 16)<<16) | (int(hexs[12:14],16)<<8) | (int(hexs[15:17], 16)) 

def net_conv(nets):
    neti = 0
    byte ='' 
    for i in range(len(nets)):
        if (nets[i]!='.'):
            byte += nets[i]
        if ((nets[i]=='.') or (i+1==len(nets))):
            neti = (neti<<8) | int(byte) 
            byte = ''
    return neti

maskbit = net_conv(campusNet_mask)

xorhb = os.urandom(12)
permb = os.urandom(12)
xorh = np.zeros(12)
perm = np.zeros(12)
for i in range(12):
    xorh[i] = xorhb[i]
    perm[i] = permb[i]
p = np.arange(12)
for i in range(12):
    n = int(perm[i] % (12-i))
    perm[i] = p[n]
    for j in range(11-n):
        p[n+j] = p[n+j+1]

def mkhash(src, sport, dest, dport): 
    res    = 0
    data   = np.zeros(12)
    srcb   = src.to_bytes(4, byteorder='big')
    dstb   = dest.to_bytes(4, byteorder='big')
    sportb = sport.to_bytes(2, byteorder='big')
    dportb = dport.to_bytes(2, byteorder='big') 
    for i in range(4):
        data[i] = srcb[i]
        data[i+4] = dstb[i]
        if (i<2):
            data[i+8] = sportb[i]
            data[i+10]= dportb[i]
    for i in range(12):
        res = ((res<<8) + (int(data[int(perm[i])]) ^ int(xorh[i]))) % 0xff100f 
#    print("flow id: ", res)
    return res

HLen    = 187187 
colli_s = 11

def FeatureExtraction(filename, interval, unit, outfile):
    ipkt = 0
    
    mean_inPktSz, mean_outPktSz = [], [] 
    var_inPktSz , var_outPktSz  = [], []
    in_bytes, ratio_in_out_bytes = [], []
    num_inFlow, num_outFlow = [], []
    num_srcIP_in, num_destIP_in, ratio_src_destIP_in, num_srcIP_out, num_destIP_out = [], [], [], [], []
    num_sport_in, num_dport_in, ratio_sport_dport_in = [], [], []
    mean_srcIP_sendN_in , var_srcIP_sendN_in = [], []
    mean_dstIP_recvN_in , var_dstIP_recvN_in = [], []
    mean_srcIP_sendN_out, var_srcIP_sendN_out= [], []
    mean_dstIP_recvN_out, var_dstIP_recvN_out= [], []
#    perc_inTCP, perc_outTCP = [], []
    # entropy
    H_srcIP_in, H_destIP_in, H_srcIP_out, H_destIP_out = [], [], [], []
    H_sport_in, H_dport_in , H_sport_out, H_dport_out  = [], [], [], []

    mean_pkts_inFlow  , var_pkts_inFlow  = [], []
    mean_pkts_outFlow , var_pkts_outFlow = [], []
    mean_SYN_destIP_in, var_SYN_destIP_in= [], []
    mean_SYN_srcIP_out, var_SYN_srcIP_out= [], []
    mean_RST_srcIP_out, var_RST_srcIP_out= [], []

#    inbound_b, outbount_b = [], []  
    # within an interval
    in_pkt_sz, out_pkt_sz = [], [] 
    in_tcp_sz, out_tcp_sz = 0, 0
    in_sport, in_dport   = np.zeros(65536), np.zeros(65536)
    out_sport, out_dport = np.zeros(65536), np.zeros(65536)
     
    in_src_2_destIP  = {"id": np.zeros(HLen), "srcIP": np.zeros(HLen)}
    in_dest_2_srcIP  = {"id": np.zeros(HLen), "dstIP": np.zeros(HLen)}
    out_src_2_destIP = {"id": np.zeros(HLen), "srcIP": np.zeros(HLen)}
    out_dest_2_srcIP = {"id": np.zeros(HLen), "dstIP": np.zeros(HLen)} 
    inFlowHash_t  = {"id": np.zeros(HLen), "packets": np.zeros(HLen)}
    outFlowHash_t = {"id": np.zeros(HLen), "packets": np.zeros(HLen)}
    inIPHash_t = {"srcIP": np.zeros(HLen), "srcIP_pkts": np.zeros(HLen), "destIP": np.zeros(HLen), "destIP_pkts": np.zeros(HLen), "destIP_SYN": np.zeros(HLen)}
    outIPHash_t = {"srcIP": np.zeros(HLen), "srcIP_pkts": np.zeros(HLen), "destIP": np.zeros(HLen), "destIP_pkts": np.zeros(HLen), "srcIP_SYN": np.zeros(HLen), "srcIP_RST": np.zeros(HLen)}

    sum_len, sum_pkt, itv = 0, 0, 0
    file_id = 0 
    pkts = pcap.pcap(filename[file_id]).readpkts()
    cutoff = []

    while (True):
        if (((unit=='b' and sum_len>interval) or (unit=='p' and (sum_pkt>0 and sum_pkt%interval==0))) or
            ((ipkt==len(pkts)-1) and (file_id==len(filename)-1))):
            print(sum_pkt)
            cutoff.append(ipkt)
            port_id = np.nonzero(in_sport)[0]
            num_sport_in.append(len(port_id))
            # entropy of ports
            if (len(port_id)==0):
                H_sport_in.append(0)
            else:
                prob_port = in_sport[port_id] / np.sum(in_sport[port_id])
                H_sport_in.append(scipy.stats.entropy(prob_port))
            port_id = np.nonzero(out_sport)[0]
            if (len(port_id)==0):
                H_sport_out.append(0)
            else:
                prob_port = out_sport[port_id] / np.sum(out_sport[port_id])
                H_sport_out.append(scipy.stats.entropy(prob_port))
            port_id = np.nonzero(in_dport)[0]
            num_dport_in.append(len(port_id))
            if (len(port_id)==0):
                H_dport_in.append(0)
                ratio_sport_dport_in.append(MAX_Ratio)
            else:
                prob_port = in_dport[port_id] / np.sum(in_dport[port_id])
                H_dport_in.append(scipy.stats.entropy(prob_port))
                ratio_sport_dport_in.append( num_sport_in[-1]/num_dport_in[-1])
            port_id = np.nonzero(out_dport)[0]
            if (len(port_id)==0):
                H_dport_out.append(0)
            else:
                prob_port = out_dport[port_id] / np.sum(out_dport[port_id])
                H_dport_out.append(scipy.stats.entropy(prob_port))

            # packets mean, variance
            in_pkt_sz  = np.array(in_pkt_sz)
            out_pkt_sz = np.array(out_pkt_sz)
            if (np.sum(in_pkt_sz)==0):
                mean_inPktSz.append(0)
                var_inPktSz.append(0)
                in_bytes.append(0)
            else:
                mean_inPktSz.append(np.mean(in_pkt_sz))
                var_inPktSz.append(np.math.sqrt(np.var(in_pkt_sz)))
                in_bytes.append(np.sum(in_pkt_sz))
            if (np.sum(out_pkt_sz)==0):
                mean_outPktSz.append(0)
                ratio_in_out_bytes.append(MAX_Ratio)
                var_outPktSz.append(0)
            else:
                mean_outPktSz.append(np.mean(out_pkt_sz))
                ratio_in_out_bytes.append(np.sum(in_pkt_sz) / np.sum(out_pkt_sz))
                var_outPktSz.append(np.math.sqrt(np.var(out_pkt_sz)))
            flowid = np.nonzero(inFlowHash_t["id"])[0]
            num_inFlow.append(len(flowid))
            if (num_inFlow[-1]==0):
                mean_pkts_inFlow.append(0)
                var_pkts_inFlow.append(0)
            else:
                mean_pkts_inFlow.append(np.mean(inFlowHash_t["packets"][flowid]))
                var_pkts_inFlow.append(np.math.sqrt(np.var(inFlowHash_t["packets"][flowid])))
            flowid = np.nonzero(outFlowHash_t["id"])[0]
            num_outFlow.append(len(flowid))
            if (num_outFlow[-1]==0):
                mean_pkts_outFlow.append(0)
                var_pkts_outFlow.append(0)
            else:
                mean_pkts_outFlow.append(np.mean(outFlowHash_t["packets"][flowid]))
                var_pkts_outFlow.append(np.math.sqrt(np.var(outFlowHash_t["packets"][flowid])))

            # entropy of ip address
            IP_id = np.nonzero(inIPHash_t["srcIP"])[0]
            num_srcIP_in.append(len(IP_id))
            if (num_srcIP_in[-1]==0):
                H_srcIP_in.append(0)
            else:
                prob_ip = inIPHash_t["srcIP_pkts"][IP_id] / sum(inIPHash_t["srcIP_pkts"][IP_id])
                H_srcIP_in.append(scipy.stats.entropy(prob_ip))
            IP_id = np.nonzero(inIPHash_t["destIP"])[0]
            num_destIP_in.append(len(IP_id))
            if (num_destIP_in[-1]==0):
                H_destIP_in.append(0)
                ratio_src_destIP_in.append(MAX_Ratio)
            else:
                prob_ip = inIPHash_t["destIP_pkts"][IP_id] / sum(inIPHash_t["destIP_pkts"][IP_id])
                H_destIP_in.append(scipy.stats.entropy(prob_ip))
                ratio_src_destIP_in.append(num_srcIP_in[-1]/num_destIP_in[-1])
            if (len(inIPHash_t["destIP_SYN"][IP_id])==0):
                mean_SYN_destIP_in.append(0)
                var_SYN_destIP_in.append(0)
            else:
                mean_SYN_destIP_in.append(np.mean(inIPHash_t["destIP_SYN"][IP_id]))
                var_SYN_destIP_in.append(np.math.sqrt(np.var(inIPHash_t["destIP_SYN"][IP_id])))

            IP_id = np.nonzero(outIPHash_t["srcIP"])[0]
            num_srcIP_out.append(len(IP_id))
            if (num_srcIP_out[-1]==0):
                H_srcIP_out.append(0)
            else:
                prob_ip = outIPHash_t["srcIP_pkts"][IP_id] / sum(outIPHash_t["srcIP_pkts"][IP_id])
                H_srcIP_out.append(scipy.stats.entropy(prob_ip))
            if (len(outIPHash_t["srcIP_SYN"][IP_id])==0):
                var_SYN_srcIP_out.append(0)
                mean_SYN_srcIP_out.append(0)
            else:
                var_SYN_srcIP_out.append(np.math.sqrt(np.var(outIPHash_t["srcIP_SYN"][IP_id])))
                mean_SYN_srcIP_out.append(np.mean(outIPHash_t["srcIP_SYN"][IP_id]))
            if (len(outIPHash_t["srcIP_RST"][IP_id])==0):
                var_RST_srcIP_out.append(0)
                mean_RST_srcIP_out.append(0)
            else:
                var_RST_srcIP_out.append(np.math.sqrt(np.var(outIPHash_t["srcIP_RST"][IP_id])))
                mean_RST_srcIP_out.append(np.mean(outIPHash_t["srcIP_RST"][IP_id]))
            IP_id = np.nonzero(outIPHash_t["destIP"])[0]
            num_destIP_out.append(len(IP_id))
            if (num_destIP_out[-1]==0):
                H_destIP_out.append(0)
            else:
                prob_ip = outIPHash_t["destIP_pkts"][IP_id] / sum(outIPHash_t["destIP_pkts"][IP_id])
                H_destIP_out.append(scipy.stats.entropy(prob_ip))
            # IP communication table
            IP_id = np.nonzero(in_src_2_destIP["id"])[0]
            ip_t = []
            src = in_src_2_destIP["srcIP"][IP_id]
            src = src[np.argsort(in_src_2_destIP["id"][IP_id])]
            cnt, i = len(src), 0
            while i < cnt:
                ip_t.append(1)
                i += 1
                while (i<cnt and src[i]==src[i-1]):
                    ip_t[-1] += 1
                    i += 1
            if (len(ip_t)==0):
                mean_srcIP_sendN_in.append(0)
                var_srcIP_sendN_in.append(0)
            else:
                mean_srcIP_sendN_in.append(np.mean(ip_t))
                var_srcIP_sendN_in.append(np.math.sqrt(np.var(ip_t)))

            IP_id = np.nonzero(out_src_2_destIP["id"])[0]
            ip_t = []
            src = out_src_2_destIP["srcIP"][IP_id]
            src = src[np.argsort(out_src_2_destIP["id"][IP_id])]
            cnt, i = len(src), 0
            while i < cnt:
                ip_t.append(1)
                i += 1
                while (i<cnt and src[i]==src[i-1]):
                    ip_t[-1] += 1
                    i += 1
            if (len(ip_t)==0):
                mean_srcIP_sendN_out.append(0)
                var_srcIP_sendN_out.append(0)
            else:
                mean_srcIP_sendN_out.append(np.mean(ip_t))
                var_srcIP_sendN_out.append(np.math.sqrt(np.var(ip_t)))

            IP_id = np.nonzero(in_dest_2_srcIP["id"])[0]
            ip_t = []
            dst = in_dest_2_srcIP["dstIP"][IP_id]
            dst = dst[np.argsort(in_dest_2_srcIP["id"][IP_id])]
            cnt, i = len(dst), 0
            while i < cnt:
                ip_t.append(1)
                i += 1
                while (i<cnt and dst[i]==dst[i-1]):
                    ip_t[-1] += 1
                    i += 1
            if (len(ip_t)==0):
                mean_dstIP_recvN_in.append(0)
                var_dstIP_recvN_in.append(0)
            else:
                mean_dstIP_recvN_in.append(np.mean(ip_t))
                var_dstIP_recvN_in.append(np.math.sqrt(np.var(ip_t)))

            IP_id = np.nonzero(out_dest_2_srcIP["id"])[0]
            ip_t = []
            dst = out_dest_2_srcIP["dstIP"][IP_id]
            dst = dst[np.argsort(out_dest_2_srcIP["id"][IP_id])]
            cnt, i = len(dst), 0
            while i < cnt:
                ip_t.append(1)
                i += 1
                while (i<cnt and dst[i]==dst[i-1]):
                    ip_t[-1] += 1
                    i += 1
            if (len(ip_t)==0):
                mean_dstIP_recvN_out.append(0)
                var_dstIP_recvN_out.append(0)
            else:
                mean_dstIP_recvN_out.append(np.mean(ip_t))
                var_dstIP_recvN_out.append(np.math.sqrt(np.var(ip_t)))

            inFlowHash_t  = {"id": np.zeros(HLen), "packets": np.zeros(HLen)}
            outFlowHash_t = {"id": np.zeros(HLen), "packets": np.zeros(HLen)}
            inIPHash_t = {"srcIP": np.zeros(HLen), "srcIP_pkts": np.zeros(HLen), "destIP": np.zeros(HLen), "destIP_pkts": np.zeros(HLen), "destIP_SYN": np.zeros(HLen)}
            outIPHash_t = {"srcIP": np.zeros(HLen), "srcIP_pkts": np.zeros(HLen), "destIP": np.zeros(HLen), "destIP_pkts": np.zeros(HLen), "srcIP_SYN": np.zeros(HLen), "srcIP_RST": np.zeros(HLen)}
            in_src_2_destIP  = {"id": np.zeros(HLen), "srcIP": np.zeros(HLen)}
            in_dest_2_srcIP  = {"id": np.zeros(HLen), "dstIP": np.zeros(HLen)}
            out_src_2_destIP = {"id": np.zeros(HLen), "srcIP": np.zeros(HLen)}
            out_dest_2_srcIP = {"id": np.zeros(HLen), "dstIP": np.zeros(HLen)}
            in_sport, in_dport   = np.zeros(65536), np.zeros(65536)
            out_sport, out_dport = np.zeros(65536), np.zeros(65536)
            in_pkt_sz, out_pkt_sz = [], []
            in_tcp_sz, out_tcp_sz = 0, 0
            sum_len = 0
            itv = itv + 1
            print("interval: ", itv)

        if (ipkt==len(pkts)):
            file_id += 1
            if (file_id==100):#len(filename)):
                dic = {'mean(B_inPkt)': mean_inPktSz, 'var(B_inPkt)': var_inPktSz}
                dic.update({'mean(B_outPkt)': mean_outPktSz, 'var(B_outPkt)': var_outPktSz})
                dic.update({'in_bytes': in_bytes, 'in/out_bytes': ratio_in_out_bytes})
                dic.update({'srcIP_in': num_srcIP_in, 'srcIP/destIP_in': ratio_src_destIP_in})
                dic.update({'H(pkt_srcIP_in)': H_srcIP_in, "H(pkt_destIP_in)": H_destIP_in})
                dic.update({'H(pkt_srcIP_out)': H_srcIP_out, "H(pkt_destIP_out)": H_destIP_out})
                dic.update({'num(sport_in)': num_sport_in, 'sport/dport_in': ratio_sport_dport_in})
                dic.update({'H(pkt_sport_in)': H_sport_in, 'H(pkt_dport_in)': H_dport_in})
                dic.update({'H(pkt_sport_out)': H_sport_out,'H(pkt_dport_out)': H_dport_out})
                dic.update({'mean(srcIP_sendN_in)': mean_srcIP_sendN_in, 'var(srcIP_sendN_in)': var_srcIP_sendN_in})
                dic.update({'mean(srcIP_sendN_out)': mean_srcIP_sendN_out, 'var(srcIP_sendN_out)': var_srcIP_sendN_out})
                dic.update({'mean(destIP_recvN_in)': mean_dstIP_recvN_in, 'var(destIP_recvN_in)': var_dstIP_recvN_in})
                dic.update({'mean(destIP_recvN_out)': mean_dstIP_recvN_out, 'var(destIP_recvN_out)': var_dstIP_recvN_out})
                dic.update({'mean(pkt_inFlow)'  : mean_pkts_inFlow  , 'var(pkt_inFlow)'  : var_pkts_inFlow})
                dic.update({'mean(pkt_outFlow)' : mean_pkts_outFlow , 'var(pkt_outFlow)' : var_pkts_outFlow})
                dic.update({'mean(SYN_destIP_in)': mean_SYN_destIP_in, 'var(SYN_destIP_in)': var_SYN_destIP_in})
                dic.update({'mean(SYN_srcIP_out)': mean_SYN_srcIP_out , 'var(SYN_srcIP_out)': var_SYN_srcIP_out})
                dic.update({'mean(RST_srcIP_out)': mean_RST_srcIP_out , 'var(RST_srcIP_out)': var_RST_srcIP_out})
                dic.update({'cutoff_ipkt': cutoff})
                print(dic) 
#                dic.update({'perc_TCP_in': perc_inTCP, 'perc_TCP_out': perc_outTCP})
                df = pd.DataFrame(data=dic)
                df.to_csv(outfile+'.20k.'+str(file_id), index=False, sep=' ')
                print(df.head())
                break
            pkts = pcap.pcap(filename[file_id]).readpkts()
            print("readin: ", filename[file_id])
            ipkt = 0

        pkt = pkts[ipkt][1]
        if (ord(pkt[12:13])!= 8): # only check IPv4
            ipkt = ipkt + 1
            continue
        sum_pkt = sum_pkt + 1
        plen    = struct.unpack('!H', pkt[16:18])[0] + 14
#        print(plen)
        sum_len += plen 
        srcIP   = struct.unpack('!I', pkt[26:30])[0] 
        destIP  = struct.unpack('!I', pkt[30:34])[0] 

        sport, dport, syn, rst, tcp, udp = 0, 0, 0, 0, False, False
        if (ord(pkt[23:24])==6):  # TCP
            try:
                sport = struct.unpack('!H', pkt[34:36])[0] 
                dport = struct.unpack('!H', pkt[36:38])[0] 
                syn   = (ord(pkt[47:48]) & 0x02 == 0x02) 
                rst   = (ord(pkt[47:48]) & 0x14 == 0x14)
                tcp   = True
            except IndexError:
                print("weird packet...")
        elif (ord(pkt[23:24])==17):  # UDP
            try:
                sport = struct.unpack('!H', pkt[34:36])[0] 
                dport = struct.unpack('!H', pkt[36:38])[0]
                udp   = True
            except IndexError:
                print("weird packet...")

        inbound = ((destIP & maskbit) == destIP)
        if (inbound):  # incoming traffic
            in_pkt_sz.append(plen)
            if (sport and dport):
                in_sport[sport] += 1
                in_dport[dport] +=1
                if (tcp):
                    in_tcp_sz += in_pkt_sz[-1]
            ip  = srcIP % HLen
            iph = ip
            while (inIPHash_t["srcIP"][iph]!=0 and inIPHash_t["srcIP"][iph]!=srcIP and iph+colli_s!=ip):
                iph = (iph+colli_s) % HLen
            if (iph+colli_s!=ip):
                if (inIPHash_t["srcIP"][iph]==0):
                    inIPHash_t["srcIP"][iph] = srcIP
                inIPHash_t["srcIP_pkts"][iph] += 1
            ip  = destIP % HLen
            iph = ip
            while (inIPHash_t["destIP"][iph]!=0 and inIPHash_t["destIP"][iph]!=destIP and iph+colli_s!=ip):
                iph = (iph + colli_s) % HLen
            if (iph+colli_s!=ip):
                if (inIPHash_t["destIP"][iph]==0):
                    inIPHash_t["destIP"][iph] = destIP
                inIPHash_t["destIP_pkts"][iph]+= 1
                inIPHash_t["destIP_SYN"][iph] += syn 
#            print(srcIP, sport, destIP, dport)

            # IP Flow Table
            ipf = (srcIP << 32) | destIP
            ip  = mkhash(srcIP, 0, destIP, 0) % HLen
            iph = ip
            while (in_src_2_destIP["id"][iph]!=0 and in_src_2_destIP["id"][iph]!= ipf and iph+colli_s!=ip):
                iph = (iph+colli_s) % HLen
            if (in_src_2_destIP["id"][iph]==0):
                in_src_2_destIP["id"][iph] = ipf 
                in_src_2_destIP["srcIP"][iph] = srcIP
            ipf = ((destIP << 32) | srcIP)
            iph = ip
            while (in_dest_2_srcIP["id"][iph]!=0 and in_dest_2_srcIP["id"][iph]!= ipf and iph+colli_s!=ip):
                iph = (iph+colli_s) % HLen
            if (in_dest_2_srcIP["id"][iph]==0):
                in_dest_2_srcIP["id"][iph] = ipf 
                in_dest_2_srcIP["dstIP"][iph] = destIP
             
            flow = mkhash(srcIP, sport, destIP, dport)  
            fid  = flow % HLen
            fhd  = fid
            while (inFlowHash_t["id"][fhd]!=0 and inFlowHash_t["id"][fhd]!=flow and fhd+colli_s!=fid):
                fhd = (fhd+colli_s) % HLen
            if (fhd+colli_s != fid):
                if (inFlowHash_t["id"][fhd]==0):
                    inFlowHash_t["id"][fhd] = flow
                inFlowHash_t["packets"][fhd] += 1

        else:  # outgoing traffic
            out_pkt_sz.append(plen)
            if (sport and dport):
                out_sport[sport] += 1
                out_dport[dport] += 1
                if (tcp):
                    out_tcp_sz += out_pkt_sz[-1]
            ip  = srcIP % HLen
            iph = ip
            while (outIPHash_t["srcIP"][iph]!=0 and outIPHash_t["srcIP"][iph]!=srcIP and iph+colli_s!=ip):
                iph = (iph+colli_s) % HLen
            if (iph+colli_s!=ip):
                if (outIPHash_t["srcIP"][iph]==0):
                    outIPHash_t["srcIP"][iph] = srcIP
                outIPHash_t["srcIP_pkts"][iph] += 1
                outIPHash_t["srcIP_SYN"][iph]  += syn
                outIPHash_t["srcIP_RST"][iph]  += rst
            ip    = destIP % HLen
            iph   = ip      
            while (outIPHash_t["destIP"][iph]!=0 and outIPHash_t["destIP"][iph]!=destIP and iph+colli_s!=ip):
                iph = (iph+colli_s) % HLen
            if (iph+colli_s!=ip):
                if (outIPHash_t["destIP"][iph]==0):
                    outIPHash_t["destIP"][iph] = destIP
                outIPHash_t["destIP_pkts"][iph]+= 1
#            print(srcIP, sport, destIP, dport)

            # IP Flow Table
            ipf = (srcIP << 32) | destIP
            ip  = mkhash(srcIP, 0, destIP, 0) % HLen
            iph = ip
            while (out_src_2_destIP["id"][iph]!=0 and out_src_2_destIP["id"][iph]!= ipf and iph+colli_s!=ip):
                iph = (iph+colli_s) % HLen
            if (out_src_2_destIP["id"][iph]==0):
                out_src_2_destIP["id"][iph] = ipf
                out_src_2_destIP["srcIP"][iph] = srcIP

            ipf = (destIP << 32) | srcIP
            iph = ip
            while (out_dest_2_srcIP["id"][iph]!=0 and out_dest_2_srcIP["id"][iph]!= ipf and iph+colli_s!=ip):
                iph = (iph+colli_s) % HLen
            if (out_dest_2_srcIP["id"][iph]==0):
                out_dest_2_srcIP["id"][iph] = ipf
                out_dest_2_srcIP["dstIP"][iph] = destIP

            flow = mkhash(srcIP, sport, destIP, dport)
            fid  = flow % HLen
            fhd  = fid
            while (outFlowHash_t["id"][fhd]!=0 and outFlowHash_t["id"][fhd]!=flow and fhd+colli_s!=fid):
                fhd = (fhd+colli_s) % HLen
            if (fhd+colli_s!=fid): 
                if (outFlowHash_t["id"][fhd]==0):
                    outFlowHash_t["id"][fhd] = flow
                outFlowHash_t["packets"][fhd] +=1

        ipkt = ipkt + 1
 
filename = os.listdir('/data/fermi-cms/20200226-1305-1358/')
#filename = ['/data/IDS/UNB-IDS2017/PCAP/Wednesday-DoS.pcap']
#filename = [f for f in os.listdir('./pcap/fermi-campus/') if re.match(r'1574362597035-6220.*.pcapng', f)]
outfile  = 'feature_csv/' + filename[0][:-4]+'csv'
#print("outfile: ", outfile)
for i in range(len(filename)):
    filename[i] = '/data/fermi-cms/20200226-1305-1358/'+filename[i]
#outfile = 'Wednesday-DoS.csv'
print(filename)
FeatureExtraction(filename, 200000, 'p', outfile)  # interval: bytes