--[[   
  
   Copyright (C) 2016 Netronome Systems, Inc. All rights reserved.

   Description : Generates a bursty traffic pattern using the rules specified
                 below.

   -- All packets generated will share the same source IP, source MAC and size
      profile (fixed or IMIX).
   -- The destination IP and MAC are selected sequentially in the following 
      pattern:

   a) The stream number is set to 0.
   b) The stream base is calculated by multiplying the stream number with the 
      MAC and IP stride.
   c) The burst number is set to 0.
   d) The flow number is set to 0.
   e) A packet is generated with the destination IP and MAC set to "stream 
      base" + "flow number".
   f) The flow number is increased, if it is less than the number of flows per 
      stream, loop back to (e), else continue to (g).
   g) The burst number is increased, if it is less than the number of bursts 
      per stream, loop back to (d), else continue to (h).
   h) The stream base is incremented, if it is less than the number of streams,
      loop back to (a).

   -- This results in a set of streams that are separated by the MAC and IP 
      stride and repeat in bursts. The destination IP and MAC increase in 
      lock-step.
   -- Packet size can be constant, or selected from a 7:4:1 IMIX profile with 
      packet sizes 1514, 570, and 64, respectively. 
--]]

local libmoon  = require "libmoon"
local mg       = require "moongen"
local moongen  = require "dpdk"      
local dpdkc    = require "dpdkc"       
local memory   = require "memory"
local device   = require "device"
local ffi      = require "ffi"
local log      = require "log"
local bit      = require "bit"

local lshift    , rshift    , band    , bswap     , bor    , ror    = 
      bit.lshift, bit.rshift, bit.band, bit.bswap , bit.bor, bit.ror

-------------------------------------------------------------------------------
-- Table for RX stats:
--
--  bytes         : Total number of bytes received.
--  prevBytes     : Number of bytes received at the previous iteration.
--  packets       : Total number of packets received.
--  prevPackets   : Number of packets received at the previous iteration.
--  dropped       : Number of received packets which have been dropped.
--  tstampPackets : Number of received packets with a timestamp.
--  meanLatency   : Mean latency of received packets with timestamps.
--  elapsedTime   : Time elapsed since the start of the run.
--  deltaTime     : Time since the last iteration.
-------------------------------------------------------------------------------
local statsRx = {
  bytes         = 0,
  prevBytes     = 0,
  packets       = 0,
  prevPackets   = 0,
  dropped       = 0,
  tstampPackets = 0,
  meanLatency   = 0,
  elapsedTime   = 0,
  deltaTime     = 0
}

-------------------------------------------------------------------------------
-- Table for TX stats:
--
--  bytes         : Total number of bytes sent.
--  prevBytes     : Number of bytes sent at the previous iteration.
--  packets       : Total number of packets sent.
--  prevPackets   : Number of packets sent at the previous iteration.
--  bursts        : Number of bursts sent.
--  short         : Number of generated by not sent packets.
--  elapsedTime   : Time elapsed since the start of the run.
--  deltaTime     : Time since the last iteration.
--------------------------------------------------------------------------------
local statsTx = {
  bytes         = 0,
  prevBytes     = 0,
  packets       = 0,
  prevPackets   = 0,
  bursts        = 0,
  short         = 0,
  elapsedTime   = 0,
  deltaTime     = 0
}

-------------------------------------------------------------------------------
-- Table for command line configuration parameters, the following are options:
--
--  rxSlavePorts    : Ports to use for the RX slaves.
--  txSlavePorts    : Ports to use for the TX slaves.
--  rxDescs         : Number of RX descriptors.
--  txDescs         : Number of TX descriptors.
--  numberOfStreams : Number of streams to create.
--  flowsPerStream  : The number of flows in each of the streams.
--  burstsPerStream : The number of bursts to send for each stream.
--  totalFlows      : The totla number of flows to generate.
--  paramDisplay    : The number of milliseconds to display the parameters for.
--  statsDisplay    : The amount of time between each device stats display.
--  txDelta         : Amount of time to pass before sending a burst.
--  packetSize      : The size of each packet, in bytes.
--  mustImix        : If IMIXing must be used to determine packet sizes.
--  iterations      : Number of iterations to run for.
--  timeout         : Number of seconds to run for.
--  fileprefix      : The prefix of the file to write to.
--  writemode       : Write mode: 0-off, 1-print, 1-globalfile, 2-percorefile
--  rest            : Parameters used to generate the traffic pattern.
-------------------------------------------------------------------------------
local clParams = {
    rxSlavePorts    = {}                      ,
    txSlavePorts    = {}                      ,
    rxDescs         = 1024                    ,
    txDescs         = 1024                    ,
    numberOfStreams = 1                       ,
    flowsPerStream  = 2047                    ,
    burstsPerStream = 1                       ,
    totalFlows      = 0                       ,
    paramDisplay    = 3000                    ,
    statsDisplay    = 1000                    ,
    txDelta         = 0                       ,
    packetSize      = 64                      ,
    mustImix        = false                   ,
    iterations      = 0                       ,
    timeout         = 0                       ,
    fileprefix      = ""                      ,
    writemode       = 1                       ,
    srcMacBase      = 0x021100000000          ,
    dstMacBase      = 0x022200000000          ,
    srcIpBase       = 3232238337              ,
    dstIpBase       = 3232241153              ,
    srcPortBase     = 50110                   ,
    dstPortBase     = 50220                   ,
    srcPortVary     = 1                       ,
    dstPortVary     = 0                       ,
    srcIpVary       = 0                       ,
    dstIpVary       = 1                       ,
    srcMacVary      = 0                       ,
    dstMacVary      = 1                       ,
    srcPortStride   = 0                       ,
    dstPortStride   = 0                       ,
    srcIpStride     = 0                       ,
    dstIpStride     = 0                       ,
    srcMacStride    = 0                       ,
    dstMacStride    = 0 
}

-------------------------------------------------------------------------------
-- Defines parameters for timing functionality.
--
--  prevtsc  : The previous clock count.
--  currtsc  : The current clock count.
--  difftsc  : The difference between the clock counts.
-------------------------------------------------------------------------------
local timeParams = {
    prevtsc  = 0 ,
    currtsc  = 0 , 
    difftsc  = 0
}

-------------------------------------------------------------------------------
-- Table for constants:
--
--  maxSlavePort      : Maximum port number a slave may use.
--  maxSlaves         : Maximum number of total slaves (limited by cores).
--  maxFlowsPerStream : Maximum number of flows per stream.
--  maxPacketSize     : Maximum size of a packet.
--  rxBurstSize       : The size of each of the RX burst.
--  txBurstSize       : The size of each of the TX bursts.
--  receiveTimeout    : Time to try receive for (ms)
--  ethHeaderLength   : Number of bytes in ethernet header.
--  ipHeaderLength    : Number of bytes in ip header.
--  udpHeaderLength   : Number of bytes in udp header
--  ethTypeIpv4       : Type code for ethernet ipv4.
--  printTimePassPcnt : % of print time which must pass since last print.
--  defaultPayload    : Default payload to use for packets.
-------------------------------------------------------------------------------
constants = {
    maxSlavePort      = 32    ,
    maxSlaves         = 32    ,
    maxFlowsPerStream = 2047  ,
    maxPacketSize     = 64    ,
    maxPacketSize     = 1522  ,
    rxBurstSize       = 128   ,
    txBurstSize       = 32    ,
    receiveTimeout    = 100   ,
    ethHeaderLength   = 14    ,
    ipHeaderLength    = 20    , 
    udpHeaderLength   = 8     ,
    ethTypeIpv4       = 0x0800,
    printTimePassPcnt = 0.8   ,
    defaultPayload    = "\x01\x02 MoonGen Payload"
}

-- Start time of the appliction. This is used to sync 
-- the printing and logging accross the slaves.
local globalStartTime = mg.getTime()

-------------------------------------------------------------------------------
-- Master function to do the slave setup and invoke the slave instances.
-------------------------------------------------------------------------------
function master(...)
    local continue = parseArgs(clParams, ...)
    if continue == false then 
        os.exit() 
    end

    local totalSlaves = #clParams.rxSlavePorts + #clParams.txSlavePorts
    if totalSlaves == 0 then 
        print("No slaves requested. Packetgen will now exit.")
        os.exit()
    end
    if totalSlaves > constants.maxSlaves then 
        print(string.format("Too many slaves requested:\n %u requested, " ..
            "%u maximum.\n Packetgen will now exit.", 
            totalSlaves, constants.maxSlaves)
        )
    end

    -- Create a list of ports, default to the tx ports:
    local portList = deepCopy(clParams.txSlavePorts)

    -- Check which of the rx ports were already added as tx ports,
    -- and add any of the rx ports which are not already in the list:
    for i, rxPortId in ipairs(clParams.rxSlavePorts) do
        local canAdd = true
        for _, portId in ipairs(portList) do
            if rxPortId == portId then
                canAdd = false
                break
            end
        end
        if canAdd then
            portList[#portList + 1] = rxPortId
        end
    end

    -- Table of MoonGen devices:
    local devices = {}
    -- Configure the devices:
    for _, port in ipairs(portList) do
        local deviceIdx = #devices + 1
        devices[deviceIdx] = device.config{
            port     = port            , 
            rxQueues = 1               , -- Only 1 supported for now.
            txQueues = 1               , -- Only 1 supported for now.
            rxDescs  = clParams.rxDescs,
            txDescs  = clParams.txDescs 
        }
    end

    -- If writing globally, write the header for the global file:
    if clParams.writemode == 2 then 
        writeStatsHeader(clParams.fileprefix .. ".txt")
    end

    -- Display the parameters:
    clParams:print()
    mg.sleepMillis(clParams.paramDisplay)

    local slaveId = 0
    -- Launch the slaves:
    for i, portId in ipairs(portList) do
        -- Check if the port must be used as a TX slave:
        for _, txPortId in ipairs(clParams.txSlavePorts) do
            if txPortId == portId then
                libmoon.startTask("txSlave", devices[i], portId, slaveId, clParams)
                slaveId = slaveId + 1
                break
            end
        end
        -- Check if the port must be used as a RX slave:
        for _, rxPortId in ipairs(clParams.rxSlavePorts) do
            if rxPortId == portId then
                libmoon.startTask("rxSlave", devices[i], portId, slaveId, clParams)
                slaveId = slaveId + 1
                break
            end
        end
    end

    mg.waitForTasks()
end 

---- RX Functionality ---------------------------------------------------------

-------------------------------------------------------------------------------
-- Slave function to perform RX.
-- device   : The MoonGen device to sent the packets from.
-- portId   : The port the device is configured to use.
-- slaveId  : The identifier (index) of the slave.
-- clParams : The command line parameters.
-------------------------------------------------------------------------------
function rxSlave(device, portId, slaveId, clParams)
  print(string.format("Launching RX slave: %u, Core: %u, Port: %u", slaveId, mg:getCore(), portId))

  -- Variables for receiving:
  local rxPackets = 0                  
  local rxQueue   = device:getRxQueue(0) 
  local rxBurst   = memory.bufArray(constants.rxBurstSize)
  local latency   = 0
  local latDelta  = 0

  -- Variables for logging, printing, stopping:
  local state         = "canrun"   
  local stats         = deepCopy(statsRx) 
  local startTime     = mg.getTime()    
  local lastPrintTime = 0
  local iteration     = 0

  -- Create the filename of the file to write to.
  if clParams.writemode == 2 then
      outFile = clParams.fileprefix .. ".txt"
  elseif clParams.writemode == 3 then
      -- Prer core mode - create file and write header:
      outFile = clParams.fileprefix                 .. 
                "-c" .. tostring(mg:getCore()) .. 
                "-p" .. tostring(portId) .. ".txt";
      writeStatsHeader(outFile)
  end

  while state == "canrun" and mg.running() do
      rxPackets     = rxQueue:tryRecv(rxBurst, constants.receiveTimeout)
      stats.packets = stats.packets + rxPackets
      stats.dropped = stats.dropped + rxPackets

      for i = 1, rxPackets do
          stats.bytes = stats.bytes + rxBurst[i].pkt_len
          latency     = calculateLatency(rxBurst[i])

          if latency ~= 0 then 
              stats.tstampPackets = stats.tstampPackets + 1
              latDelta            = latency - stats.meanLatency
              stats.meanLatency   = stats.meanLatency + 
                                    (latDelta / stats.tstampPackets)
          end
      end
      simpleDrop(rxBurst)

      -- Update termination params:
      iteration         = iteration + 1
      stats.deltaTime   = (mg.getTime() - startTime) - stats.elapsedTime
      stats.elapsedTime = mg.getTime() - startTime
      
      -- If printing or logging must be done:
      if clParams.writemode > 0 then 
          -- To make sure that enough time has passed since logging 
          -- and printing was done for the core.
          sufficientTimeSincePrint = 
            (stats.elapsedTime - lastPrintTime) >= 
            (constants.printTimePassPcnt * clParams.statsDisplay / 1000)

          -- Printing and logging:
          if canPrint(slaveId, clParams) and sufficientTimeSincePrint then 
              lastPrintTime = mg.getTime() - startTime
              if clParams.writemode == 1 then
                  os.execute("clear")      
                  stats:print(portId)
              else 
                  stats:write(outFile, portId)
              end
          end  
      end

      -- Update stats:
      stats.prevPackets = stats.packets
      stats.prevBytes   = stats.bytes

       -- Check for exit conditions:
      if clParams.iterations ~= 0 and iteration >= clParams.iterations then
          state = "stop"
      elseif clParams.timeout ~= 0                 and 
             stats.elapsedTime >= clParams.timeout then 
          state = "stop"
      end 
  end
  
  if clParams.writemode == 0 then 
      stats:print(portId)
  end
end

-------------------------------------------------------------------------------
-- Drops the received packets
-- rxPackets  : The received packets to drop.
-------------------------------------------------------------------------------
function simpleDrop(rxPackets) 
    rxPackets:freeAll()
end

-------------------------------------------------------------------------------
-- Gets the timestamp from a packet and then calculates the latency
-- buf  : The buffer to get the timestamp from and then calculate the latency.
-------------------------------------------------------------------------------
function calculateLatency(buf)
    local pkt       = buf:getUdpPacket()
    local cycles    = tonumber(mg:getCycles())
    local timestamp = tonumber(pkt.payload.uint64[0])
    local latency   = 0

    if timestamp ~= 0 then 
        latency = tonumber(cycles - timestamp) 
                / tonumber(mg:getCyclesFrequency())
    end
    return latency
end

-------------------------------------------------------------------------------
-- Prints RX stats
-- portId : The id of the port to print the stats for
-------------------------------------------------------------------------------
function statsRx:print(portId)
  local statsString = string.format(
    "\n+------ RX Statistics for core %3u, port %3u -------------------+"   ..
    "\n| Packets received           : %32u |"                               ..
    "\n| Packet receive rate        : %32.2f |"                             ..
    "\n| Avg packet receive rate    : %32.2f |"                             ..
    "\n| Bytes received             : %32u |"                               ..
    "\n| Byte receive rate          : %32.2f |"                             ..
    "\n| Avg byte receive rate      : %32.2f |"                             ..
    "\n| Packets dropped on receive : %32u |"                               ..
    "\n| RX mean latency            : %32.10f |"                            ..
    "\n+---------------------------------------------------------------+"   ,
    mg:getCore()                                   ,
    portId                                              ,
    self.packets                                        ,
    (self.packets - self.prevPackets) / self.deltaTime  ,
    self.packets / self.elapsedTime                     ,
    self.bytes                                          ,
    (self.bytes - self.prevBytes) / self.deltaTime      ,
    self.bytes / self.elapsedTime                       ,
    self.dropped                                        , 
    self.meanLatency
  )
  print(statsString)
end

-------------------------------------------------------------------------------
-- Writes RX a stats entry to a file. The row entry contains the following
-- fields:

--  Time, tx packets, tx packet rate, avg tx packet rate, rx packets, 
--  rx packet rate, avg rx packet rate, core, port 
--
-- filename : Name of the file to write to.
-- portId   : The port the stats are being written for.
-------------------------------------------------------------------------------
function statsRx:write(filename, portId)
    file = io.open(filename, "a")

    local time = mg.getTime() - globalStartTime

    local statsString = string.format(
        "%6.3f,%13u,%18.2f,%18.2f,%13u,%18.2f,%18.2f,%5u,%5u\n", 
        time                                                ,
        0                                                   ,
        0                                                   ,
        0                                                   ,
        self.packets                                        ,
        (self.packets - self.prevPackets) / self.deltaTime  ,
        self.packets / self.elapsedTime                     ,
        mg:getCore()                                   ,
        portId                                              
    ) 

    file:write(statsString)
    file:close()
end

---- TX Functionality ---------------------------------------------------------

-------------------------------------------------------------------------------
-- Slave function to perform TX.
-- device   : The MoonGen device to sent the packets from.
-- portId   : The port the device is configures to use.
-- slaveId  : The identifier (index) of the slave.
-- clParams : The command line parameters
-------------------------------------------------------------------------------
function txSlave(device, portId, slaveId, clParams) 
  print(string.format("Launching TX slave: %u, Core: %u, Port: %u", 
      slaveId, mg:getCore(), portId))

  local state       = "canrun"
  local stats       = deepCopy(statsTx)     
  local txQueue     = device:getTxQueue(0) 
  local timerParams = deepCopy(timeParams)
  local outFile     = ""

  -- Create the filename of the file to write to.
  if clParams.writemode == 2 then
      outFile = clParams.fileprefix .. ".txt"
  elseif clParams.writemode == 3 then
      -- Per core mode: write header
      outFile = clParams.fileprefix                 .. 
                "-c" .. tostring(mg:getCore()) .. 
                "-p" .. tostring(portId) .. ".txt";
      writeStatsHeader(outFile)
  end

  -- Allocate memory for all the streams. Each of the streams can have 
  -- a maximum of 2047 flows due to mbuf allocation limits.
  local streams = {}
  for i = 1, clParams.numberOfStreams do
    streams[i] = {
        mempool  = {},
        bufArray = nil
    }

    -- Create a mempool for the stream:
    streams[i].mempool = memory:createMemPool(
        function(buf)
            buf:getUdpPacket():fill{
                pktLength = clParams.packetSize - 4,
                ethLength = constants.ethHeaderLength
            }
        end
    )
    -- Allocate buffers from the mempool:
    streams[i].bufArray = streams[i].mempool:bufArray(clParams.flowsPerStream)
  end

  local counter = 0
  -- Modify all the packets so that their data follows the traffic pattern:
  for streamid, stream in ipairs(streams) do
      -- Subtraction of 4 is for the FCS
      local maxPacketSize = clParams.packetSize - 4
      if clParams.mustImix then 
          maxPacketSize = constants.maxPacketSize - 4
      end

      -- Allocate buffers using the max possible packet size. If using Imix
      -- then each of the sent packets will vary in size, despite the constant
      -- size allocation done here.
      stream.bufArray:alloc(maxPacketSize)

      -- Modify the packets:
      for flowid, buf in ipairs(stream.bufArray) do
          if clParams.mustImix then
              local pktSize = imixSize() - 4
              buf.pkt_len   = pktSize
              buf.data_len  = pktSize
          end 

          buildTxFrame(device.id, buf, streamid - 1, flowid - 1, clParams)
      end 
      
      -- Offload the checksum to the NIC:
      stream.bufArray:offloadUdpChecksums()
  end

  -- Do sending:
  local iteration     = 0
  local lastPrintTime = 0
  local startTime     = mg.getTime()

  while state == "canrun" and mg.running() do
      for _, stream in ipairs(streams) do
          timerParams.currtsc = tonumber(mg:getCycles())
          timerParams.difftsc = timerParams.currtsc - timerParams.prevtsc

          sendBursts(txQueue, stats, timerParams, clParams, stream)
      end

      -- Update the iteration and timeout params:
      iteration         = iteration + 1
      stats.deltaTime   = (mg.getTime() - startTime) - stats.elapsedTime
      stats.elapsedTime = mg.getTime() - startTime

      -- Printing and writing:
      if clParams.writemode > 0 then
          -- To make sure that enough time has passed since logging 
          -- and printing was done for the core. 
          sufficientTimeSincePrint = 
            ((stats.elapsedTime - lastPrintTime) >= 
             (constants.printTimePassPcnt * clParams.statsDisplay / 1000))

          -- Printing and logging:
          if canPrint(slaveId, clParams) and sufficientTimeSincePrint then 
              lastPrintTime = mg.getTime() - startTime
              if clParams.writemode == 1 then
                  os.execute("clear")      
                  stats:print(portId)
              else 
                  stats:write(outFile, portId)
              end
          end   
      end

      -- Update stats:
      stats.prevPackets = stats.packets
      stats.prevBytes   = stats.bytes

      -- Check for exit conditions:
      if clParams.iterations ~= 0 and iteration >= clParams.iterations then
          state = "stop"
      elseif clParams.timeout ~= 0                 and 
             stats.elapsedTime >= clParams.timeout then 
          state = "stop"
      end 
  end

  if clParams.writemode == 0 then 
      stats:print(portId)
  end
end

-------------------------------------------------------------------------------
-- Builds a frame for TX.
-- portId     : The port of the device to build the frame for.
-- buf        : The buffer for the packet holding the frame.
-- streamid   : The id of the stream for the frame.
-- flowid     : The id of the flow for the frame.
-- clParams   : The command line parameters.
-------------------------------------------------------------------------------
function buildTxFrame(portId, buf, streamid, flowid, clParams)
    local srcMac  = clParams.srcMacBase + (streamid * clParams.srcMacStride) 
                  + (flowid * clParams.srcMacVary)
    local dstMac  = clParams.dstMacBase + (streamid * clParams.dstMacStride) 
                  + (flowid * clParams.dstMacVary)
    local srcIp   = clParams.srcIpBase + (streamid * clParams.srcIpStride) 
                  + (flowid * clParams.srcIpVary)
    local dstIp   = clParams.dstIpBase + (streamid * clParams.dstIpStride) 
                  + (flowid * clParams.dstIpVary)
    local srcPort = band(clParams.srcPortBase
                            + (streamid * clParams.srcPortStride) 
                            + (flowid * clParams.srcPortVary),
                         0xffff)
    local dstPort = band(clParams.dstPortBase 
                            + (streamid * clParams.dstPortStride) 
                            + (flowid * clParams.dstPortVary),
                         0xffff)

    local srcMacFlipped = rshift(bswap(srcMac + 0ULL), 16)
    local dstmacFlipped = rshift(bswap(dstMac + 0ULL), 16)

    local frameSize = buf.pkt_len
    local ethLength = constants.ethHeaderLength
    local ipLength  = frameSize - ethLength 
    local udpLength = ipLength  - constants.ipHeaderLength

    local pkt = buf:getUdpPacket()

    -- ETH mod:
    pkt.eth:setType(constants.ethTypeIpv4)
    pkt.eth:setSrc(srcMacFlipped)
    pkt.eth:setDst(dstmacFlipped)

    -- IP mod:
    pkt.ip4:setLength(ipLength)
    pkt.ip4:setHeaderLength(5)  -- Number of 32 bit words : use min value.
    pkt.ip4:setProtocol(17)
    pkt.ip4:setTTL(64)
    pkt.ip4:setSrc(srcIp)
    pkt.ip4:setDst(dstIp)
    pkt.ip4:setVersion(4)

    -- UDP mod:
    pkt.udp:setLength(udpLength)
    pkt.udp:setSrcPort(srcPort)
    pkt.udp:setDstPort(dstPort)

    -- Start of payload looks as follows:
    --
    -- |  Byte 0  |  Byte 4  |  Byte 8  |  Byte 12  |
    -- ----------------------------------------------
    -- |      timestamp      |  flowid  |  streamid |
    -- ----------------------------------------------
    --
    -- The timestamp is alredy set, so set flow and stream id.
    pkt.payload.uint32[2] = flowid
    pkt.payload.uint32[3] = streamid

    -- Fill the rest of the payload
    local payLength   = udpLength - constants.udpHeaderLength
    local i           = 16             -- tstamp, flowid, streamid = 16 bytes
    local offset      = i              -- Start offset into payload bytes

    -- Copy the default payload into the packet.
    while i < payLength do
        -- NOTE: (i - offset) is to remove the 16 byte initial offset.
        pkt.payload.uint8[i] = 
            string.byte(constants.defaultPayload, i - offset + 1) or 0
        i = i + 1
    end
end

-------------------------------------------------------------------------------
-- Generates and sends a burst if enough time has passed to achieve the desired
-- packet rate, otherwise nothing happens.
-- txQueue     : The queue to send the burst on.
-- stats       : The stats to update.
-- timerParams : The parameters for the burst to send.
-- clParams    : The command line parameters.
-- stream      : The stream to send.
-------------------------------------------------------------------------------
function sendBursts(txQueue, stats , timerParams, clParams, stream)
    -- If enough time has passed to achieve the requested rate.
    if timerParams.difftsc > clParams.txDelta then 
        local nbtx = 0
        for i = 1, clParams.burstsPerStream do
            stats.bursts = stats.bursts + 1
            for _, buf in ipairs(stream.bufArray) do
                updateTimestamp(buf)
            end
            nbtx = nbtx + txQueue:send(stream.bufArray)
        end

        stats.packets = stats.packets + nbtx

        for _, buf in ipairs(stream.bufArray) do 
            -- Add four for the FCS.
            stats.bytes = 4 + stats.bytes + 
                          (clParams.burstsPerStream * buf.pkt_len)
        end

        if nbtx < constants.txBurstSize then 
            stats.short = stats.short + 1
        end

        timerParams.prevtsc = timerParams.currtsc
    end
end

-------------------------------------------------------------------------------
-- Updates the timestamp for a packet. The first 8 bytes of the packet payload 
-- are used for the timestamp.
-- buf  : The buffer to update the timestamp of.
-------------------------------------------------------------------------------
function updateTimestamp(buf)
    local pkt             = buf:getUdpPacket()
    local cycles          = mg:getCycles()
    pkt.payload.uint64[0] = cycles
end

-------------------------------------------------------------------------------
-- Prints TX stats
-- portId : The id of the port to print the stats for
-------------------------------------------------------------------------------
function statsTx:print(portId)
  local statsString = string.format(
    "\n+------ TX Statistics for core %3u, port %3u -------------------+"   ..
    "\n| Packets sent               : %32u |"                               ..
    "\n| Packet send rate           : %32.2f |"                             ..
    "\n| Avg packet send rate       : %32.2f |"                             ..
    "\n| Bytes sent                 : %32u |"                               ..
    "\n| Byte send rate             : %32.2f |"                             ..
    "\n| Avg byte send rate         : %32.2f |"                             ..
    "\n| Packets short              : %32u |"                               ..
    "\n| Bursts                     : %32u |"                               ..
    "\n+---------------------------------------------------------------+"   ,
    mg:getCore()                                   ,
    portId                                              ,
    self.packets                                        ,
    (self.packets - self.prevPackets) / self.deltaTime  ,
    self.packets / self.elapsedTime                     ,
    self.bytes                                          ,
    (self.bytes - self.prevBytes) / self.deltaTime      ,
    self.bytes / self.elapsedTime                       ,
    self.short                                          ,
    self.bursts
  )
  print(statsString)
end

-------------------------------------------------------------------------------
-- Writes TX a stats entry to a file. The row entry contains the following
-- fields:

--  Time, tx packets, tx packet rate, avg tx packet rate, rx packets, 
--  rx packet rate, avg rx packet rate, core, port 
--
-- filename : Name of the file to write to.
-- portId   : The port the stats are being written for.
-------------------------------------------------------------------------------
function statsTx:write(filename, portId)
    file = io.open(filename, "a")

    local time = mg.getTime() - globalStartTime

    local statsString = string.format(
        "%6.3f,%13u,%18.2f,%18.2f,%13u,%18.2f,%18.2f,%5u,%5u\n", 
        time,
        self.packets                                        ,
        (self.packets - self.prevPackets) / self.deltaTime  ,
        self.packets / self.elapsedTime                     ,
        0                                                   ,
        0                                                   ,
        0                                                   ,
        mg:getCore()                                   ,
        portId                                              
    ) 

    file:write(statsString)
    file:close()
end

---- Utility Functions --------------------------------------------------------

-------------------------------------------------------------------------------
-- Write the header for a stats file.
-- filename : The name of the file to write the headerto.
-------------------------------------------------------------------------------
function writeStatsHeader(filename)
    file = io.open(filename, "a")

    local headerString = string.format(
        "%6s %13s %18s %18s %13s %18s %18s %5s %5s\n",
        "Time", 
        "TX Packets",
        "TX Packet Rate"    ,
        "Avg TX Packet Rate",
        "RX Packets"        ,
        "RX Packet Rate"    ,
        "Avg RX Packet Rate",
        "Core"              ,
        "Port"    
    )

    file:write(headerString)
    file:close()
end

-------------------------------------------------------------------------------
-- Determines if a core is allowed to print it's stats.
-- slaveId  : Identifier for the slave requesting to print.
-- clParams : The command line parameters
-------------------------------------------------------------------------------
function canPrint(slaveId, clParams) 
    local elapsedTime = mg.getTime() - globalStartTime
    local timePerCore = clParams.statsDisplay / 
                        (#clParams.rxSlavePorts + #clParams.txSlavePorts)

    local timeMod = math.floor(
        math.fmod(elapsedTime * 1000, clParams.statsDisplay))

    return timeMod == math.floor(timePerCore * slaveId)
end

-------------------------------------------------------------------------------
-- Prints the command line arguments.
-------------------------------------------------------------------------------
function clParams:print()
    local paramString = string.format(
      "\n+-------- Parameters ---------------------------------+"       ..
      "\n| RX Slaves          : %30u |"                                 ..
      "\n| TX Slaves          : %30u |"                                 ..
      "\n| RX Descs           : %30u |"                                 ..
      "\n| TX Descs           : %30u |"                                 ..
      "\n| Param display (s)  : %30.3f |"                               ..
      "\n| Stat display (s)   : %30.3f |"                               ..
      "\n| Number of streams  : %30u |"                                 ..
      "\n| Bursts per stream  : %30u |"                                 ..
      "\n| Flows per stream   : %30u |"                                 ..
      "\n| Total flows        : %30u |"                                 ..
      "\n| Iterations         : %30u |"                                 ..
      "\n| Timeout            : %30u |"                                 ..
      "\n| Packet size        : %30u |"                                 ..
      "\n| Using Imix size    : %30s |"                                 ..
      "\n| File prefix        : %30s |"                                 ..
      "\n| Write mode         : %30u |"                                 ..
      "\n| ETH src base       : %30s |"                                 ..
      "\n| ETH dst base       : %30s |"                                 ..
      "\n| ETH src vary       : %30s |"                                 ..
      "\n| ETH dst vary       : %30s |"                                 ..
      "\n| ETH src stride     : %30s |"                                 ..
      "\n| ETH dst stride     : %30s |"                                 ..
      "\n| IP src base        : %30s |"                                 ..
      "\n| IP dst base        : %30s |"                                 ..
      "\n| IP src vary        : %30s |"                                 ..
      "\n| IP dst vary        : %30s |"                                 ..
      "\n| IP src stride      : %30s |"                                 ..
      "\n| IP dst stride      : %30s |"                                 ..
      "\n| PORT src base      : %30u |"                                 ..
      "\n| PORT dst base      : %30u |"                                 ..
      "\n| PORT src vary      : %30u |"                                 ..
      "\n| PORT dst vary      : %30u |"                                 ..
      "\n| PORT src stride    : %30u |"                                 ..
      "\n| PORT dst stride    : %30u |"                                 ..
      "\n+-----------------------------------------------------+\n"     ,
      #self.rxSlavePorts                       ,
      #self.txSlavePorts                       ,
      self.rxDescs                             ,
      self.txDescs                             ,
      self.paramDisplay / 1000                 ,
      self.statsDisplay / 1000                 ,
      self.numberOfStreams                     ,
      self.burstsPerStream                     ,
      self.flowsPerStream                      ,
      self.totalFlows                          ,
      self.iterations                          ,
      self.timeout                             ,
      self.packetSize                          ,  
      tostring(self.mustImix)                  ,
      self.fileprefix                          ,
      self.writemode                           ,
      string.format("%012x", self.srcMacBase)  ,
      string.format("%012x", self.dstMacBase)  ,
      string.format("%012x", self.srcMacVary)  ,
      string.format("%012x", self.dstMacVary  ),
      string.format("%012x", self.srcMacStride),
      string.format("%012x", self.dstMacStride),
      formatAsIp(self.srcIpBase)               ,
      formatAsIp(self.dstIpBase)               ,
      formatAsIp(self.srcIpVary)               ,
      formatAsIp(self.dstIpVary)               ,
      formatAsIp(self.srcIpStride)             ,
      formatAsIp(self.dstIpStride)             ,
      self.srcPortBase                         ,
      self.dstPortBase                         ,
      self.srcPortVary                         ,
      self.dstPortVary                         ,
      self.srcPortStride                       ,
      self.dstPortStride  
    )

    print(paramString)
end

---- Argument Parsing ---------------------------------------------------------

-------------------------------------------------------------------------------
-- Converts a MAC address from its string representation to a numeric one, in
-- network byte order.
-- address  : The address to convert.
-------------------------------------------------------------------------------
function convertMacAddress(address)
	  local bytes = {string.match(address,
                    '(%x+)[-:](%x+)[-:](%x+)[-:](%x+)[-:](%x+)[-:](%x+)')}

    local convertedAddress = 0
    for i = 1, 6 do
        convertedAddress = convertedAddress + 
                           tonumber(bytes[#bytes + 1 - i], 16) * 256 ^ (i - 1)
    end
    return convertedAddress
end


-------------------------------------------------------------------------------
-- Formats a 32 bit value as an IP address.
-- ip   : The numeric representation of the IP address to format.
-------------------------------------------------------------------------------
function formatAsIp(ip) 
    return string.format("%d.%d.%d.%d", 
      band(rshift(ip, 24), 0xff), band(rshift(ip, 16), 0xff), 
      band(rshift(ip, 8 ), 0xff), band(ip, 0xff))
end

-------------------------------------------------------------------------------
-- Parses the command line arguments.
-- params : The a table of parameters to modify based on the command line 
--          parameters 
-- ...    : A table of command line parameters to parse.
-------------------------------------------------------------------------------
function parseArgs(params, ...)
    local command  = true  
    local args     = {...}

    for i,v in ipairs(args) do 
        if type(v) == "string" then 
            if v == "--tx-slave" or v == "-tx" then 
                local slavePort = tonumber(args[i + 1])
                if slavePort < 0 or slavePort > constants.maxSlavePort then 
                    print("Invalid TX slave port: ", slavePort)
                    printUsage()
                    return false
                end
                local canAdd = true
                for _, v in ipairs(params.txSlavePorts) do
                    if v == slavePort then
                        canAdd = false
                        print("Ignoring second specification of TX slave " ..
                              "port: ", slavePort)
                    end
                end
                if canAdd then
                    params.txSlavePorts[#params.txSlavePorts + 1] =  slavePort 
                end
            elseif v == "--rx-slave" or v == "-rx" then
                local slavePort = tonumber(args[i + 1])
                if slavePort < 0 or slavePort > constants.maxSlavePort then 
                    print("Invalid RX slave port: ", slavePort)
                    printUsage()
                    return false
                end
                local canAdd = true
                for _, v in ipairs(params.rxSlavePorts) do
                    if v == slavePort then
                        canAdd = false
                        print("Ignoring second specification of RX slave " ..
                              "port: ", slavePort)
                    end
                end
                if canAdd then
                    params.rxSlavePorts[#params.rxSlavePorts + 1] =  slavePort 
                end
            elseif v == "--tx-descs" or v == "-txd" then 
                local txDescs = tonumber(args[i + 1])
                if txDescs < 0 then 
                    print("Invalid TX descriptors: ", txDescs)
                    printUsage()
                    return false
                end
                params.txDescs = txDescs
            elseif v == "--rx-descs" or v == "-rxd" then 
                local rxDescs = tonumber(args[i + 1])
                if rxDescs < 0 then 
                    print("Invalid RX descriptors: ", rxDescs)
                    printUsage()
                    return false
                end
                params.rxDescs = rxDescs
            elseif v == "--streams" or v == "-s" then 
                local streams = tonumber(args[i + 1])
                if (streams < 0) then 
                    print("Invalid number of streams:", streams)
                    printUsage()
                    return false
                end
                params.numberOfStreams = streams
            elseif v == "--bursts-per-stream" or v == "-bps" then
                local bps = tonumber(args[i + 1])
                if (bps < 0) then 
                    print("Invalid number of bursts per stream:", bps)
                    printUsage()
                    return false
                end 
                params.burstsPerStream = bps
            elseif v == "--flows-per-stream" or v == "-fps" then 
                local fps = tonumber(args[i + 1])
                if fps < 0 or fps > constants.maxFlowsPerStream then 
                    print("Invalid number of flows per stream:", fps)
                    printUsage()
                    return false
                end 
                params.flowsPerStream = fps
            elseif v == "--param-display" or v == "-pd" then 
                local paramDisplay = tonumber(args[i + 1])
                if paramDisplay < 0 then 
                    print(string.format("Invalid param display time: %u, " ..
                          "using default: %u", 
                           paramDisplay, (params.paramDisplay / 1000)))
                else  
                    params.paramDisplay = paramDisplay * 1000
                end
            elseif v == "--stats-display" or v == "-sd" then 
                local period = tonumber(args[i + 1])
                if period < 0 or period > 86400 then 
                    print(string.format("Invalid port stats display time: " ..
                          "%u, using default: %u", period, 
                          (params.statsDisplay / 1000))
                    )
                    return false
                end 
                params.statsDisplay = period * 1000
            elseif v == "--pkt-size" or v == "-ps" then 
                local pktSize = tonumber(args[i + 1])
                if pktSize == 0 then 
                    params.mustImix   = true
                    params.packetSize = pktSize
                elseif pktSize < 64 or pktSize > 1514 then
                    print("Invalid packet size")
                    printUsage()
                    return false
                else
                    params.packetSize = pktSize
                end
            elseif v == "--timeout" or v == "-to" then 
                local timeout = tonumber(args[i + 1])
                if timeout < 0 then 
                    print("Invalid timeout period, can't be negative.")
                    printUsage()
                    return false
                else 
                    params.timeout = timeout
                end
                params.writemode = 0
            elseif v == "--iterations" or v == "-it" then
                local iterations = tonumber(args[i + 1])
                if iterations < 0 then 
                    print("Invalid number of iterations, cannot be negative.")
                    printUsage()
                    return false
                else 
                    params.iterations = iterations 
                end
                params.writemode = 0
            elseif v == "--pkts-per-sec" or v == "-pps" then 
                print("NOTE: flows per stream and burst per stream " ..
                      "should be specified first")

                local txPps = tonumber(args[i + 1]) 
                if txPps < 0 then 
                    print("Invalid packets per second, can't be negaive.")
                    printUsage()
                    return false
                end

                params.txDelta = 
                    math.floor(mg:getCyclesFrequency() / txPps) *
                      params.flowsPerStream * params.burstsPerStream
            elseif v == "--file-prefix" or v == "-fp" then
                params.fileprefix = args[i + 1]
                if params.writemode < 2 then
                    params.writemode = 2 -- write to global file
                end
            elseif v == "--write-mode" or v == "-wm" then
                local writemode = tonumber(args[i +1])
                if writemode < 0 or writemode > 3 then
                    print("Invalid write mode, must be 0 - 3.")
                    printUsage()
                    return false
                end
                params.writemode = writemode
            elseif v == "--src-port" or v == "-spt" then 
                params.srcPortBase = tonumber(args[i + 1])
            elseif v == "--dst-port" or v == "-dpt" then 
                params.dstPortBase = tonumber(args[i + 1])
            elseif v == "--src-ip" or v == "-sip" then 
                params.srcIpBase = parseIPAddress(
                    string.match(args[i + 1],
                        "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?"),
                    true)
            elseif v == "--dst-ip" or v == "-dip" then
                params.dstIpBase = parseIPAddress(
                    string.match(args[i + 1],
                        "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?"),
                    true)
            elseif v == "--src-mac" or v == "-sm" then 
                params.srcMacBase = convertMacAddress(
                    string.match(args[i + 1],
                        "%x%x:%x%x:%x%x:%x%x:%x%x:%x%x"))
            elseif v == "--dst-mac" or v == "-dm" then 
                params.dstMacBase = convertMacAddress(
                    string.match(args[i + 1],
                        "%x%x:%x%x:%x%x:%x%x:%x%x:%x%x"))
            elseif v == "--src-port-vary" or v == "-sptv" then
                params.srcPortVary = tonumber(string.match(args[i + 1], "%d+"))
            elseif v == "--dst-port-vary" or v == "-dptv" then 
                params.dstPortVary = tonumber(string.match(args[i + 1], "%d+"))
            elseif v == "--src-ip-vary" or v == "-sipv" then 
                params.srcIpVary = parseIPAddress(
                    string.match(args[i +1 ],
                        "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?"),
                    true)
            elseif v == "--dst-ip-vary" or v == "-dipv" then
                params.dstIpVary = parseIPAddress(
                    string.match(args[i + 1],
                        "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?"),
                    true)
            elseif v == "--src-mac-vary" or v == "-smv" then 
                params.srcMacVary = convertMacAddress(
                    string.match(args[i + 1],
                        "%x%x:%x%x:%x%x:%x%x:%x%x:%x%x"))
            elseif v == "--dst-mac-vary" or v == "-dmv" then 
                params.dstMacVary = convertMacAddress(
                    string.match(args[i + 1],
                        "%x%x:%x%x:%x%x:%x%x:%x%x:%x%x"))
            elseif v == "--src-port-stride" or v == "-spts" then
                params.srcPortStride = tonumber(string.match(args[i + 1], "%d+"))
            elseif v == "--dst-port-stride" or v == "-dpts" then 
                params.dstPortStride = tonumber(string.match(args[i + 1], "%d+"))
            elseif v == "--src-ip-stride" or v == "-sips" then 
                params.srcIpStride = parseIPAddress(
                    string.match(args[i +1 ],
                        "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?"),
                    true)
            elseif v == "--dst-ip-stride" or v == "-dips" then
                params.dstIpStride = parseIPAddress(
                    string.match(args[i + 1],
                        "%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?"),
                    true)
            elseif v == "--src-mac-stride" or v == "-sms" then 
                params.srcMacStride = convertMacAddress(
                    string.match(args[i + 1],
                        "%x%x:%x%x:%x%x:%x%x:%x%x:%x%x"))
            elseif v == "--dst-mac-stride" or v == "-dms" then 
                params.dstMacStride = convertMacAddress(
                    string.match(args[i + 1],
                        "%x%x:%x%x:%x%x:%x%x:%x%x:%x%x"))

            elseif v == "--help" or v == "-h" then 
                printUsage()
                return false
            end
        end
    end
    params.totalFlows = params.numberOfStreams * params.flowsPerStream

    -- Check that the strides are valid:
    checkStrideValidity(params)

    return true
end

-------------------------------------------------------------------------------
-- Checks that the strides for the parameters are valid
-- params   : The parameters whose strides validity must be checked
-------------------------------------------------------------------------------
function checkStrideValidity(params)
    if params.srcMacStride == 0 then 
        params.srsMacStride = band(params.flowsPerStream, 0xffffffffffff)  
    end
    if params.dstMacStride == 0 then 
        params.dstMacStride = band(params.flowsPerStream, 0xffffffffffff)  
    end
    if params.srcIpStride == 0 then 
        params.srcIpStride = band(params.flowsPerStream, 0xffffffff)  
    end
    if params.dstIpStride == 0 then 
        params.dstIpStride = band(params.flowsPerStream, 0xffffffff)  
    end
    if params.srcPortStride == 0 then 
        params.srcPortStride = band(params.flowsPerStream, 0xffff) 
    end  
    if params.dstPortStride == 0 then 
        params.dstPortStride = band(params.flowsPerStream, 0xffff) 
    end  
end

-------------------------------------------------------------------------------
-- Prints the example usage for the packet generation.
-------------------------------------------------------------------------------
function printUsage()
    local usage = string.format(
        "Usage is:\n Moongen packegen.lua <--dpdk-config=/path/to/config>" ..
        " <Required Args> <Optional Args> where:\n"                        ..
        " Required Args:\n\n"                                              ..
        "   --tx-slave|-tx PORT where\n"                                   ..
        "     PORT : Id of a port to use to TX\n"                          ..
        "     NOTE : This can be specified multiple times\n\n"             ..
        "   --rx-slave|-rx PORT where\n"                                   ..
        "     PORT : Id of a port to use to RX\n"                          ..
        "     NOTE : This can be specified multiple times\n\n"             ..
        " Optional Args:\n\n"                                              ..
        "   --tx-descs|-txd DESCRIPTORS where\n"                           ..
        "     DESCRIPTORS : Number of TX descriptors (default 1024)\n\n"   ..
        "   --rx-descs|-rxd DESCRIPTORS where\n"                           ..
        "     DESCRIPTORS : Number of RX descriptors (default 1024)\n\n"   ..
        "   --streams|-s NUM_STREAMS where\n"                              ..
        "     NUM_STREAMS : Number of streams (default 1)\n\n"             ..
        "   --bursts-per-stream|-bps NUM_BURSTS where\n"                   ..
        "     NUM_BURSTS : Number of bursts per stream (default 1)\n\n"    ..
        "   --flows-per-stream|-fps NUM_FLOWS where\n"                     ..
        "     NUM_FLOWS : Number of flows per stream (default 2047)\n"     ..
        "     NOTE      : 2047 is the max.\n\n"                            ..
        "   --param-display|-pd TIME where\n"                              ..
        "     TIME : Seconds to display params before running\n"           ..
        "            (default 3 seconds)\n\n"                              ..
        "   --stats-display|-sd TIME where\n"                              ..
        "     TIME Stats refresh period on each core, in seconds\n"        ..
        "     (default = 1, 0 = disable)\n"                                ..
        "     NOTE: The stats for each core are printed iteratively\n"     ..
        "           A core's stats will be printed every TIME seconds\n"   ..
        "           TIME = 3 with 3 cores will result in:\n"               ..
        "             core 1 : 0s, 3s, 6s ...\n"                           ..
        "             core 2 : 1s, 4s, 7s ...\n"                           ..
        "             core 3 : 2s, 5s, 8s ...\n\n"                         ..
        "   --pkt-size|-ps PKT_SIZE where\n"                               ..
        "     PKT_SIZE : Packet size (0 for IMIX, default 64)\n\n"         ..
        "   --timeout|-to TIME where\n"                                    ..
        "     TIME : The time (in seconds) to run for.\n"                  ..
        "            (default is to run indefinitely)\n\n"                 ..
        "   --iterations|-it ITERS where\n"                                ..
        "     ITERS : The number of TX/RX iterations to run.\n"            ..
        "             (defualt is to run indefinitely)\n\n"                ..
        "   --pps|-r RATE where\n"                                         ..
        "     RATE : Packets per second rate to attempt.\n\n"              ..
        "   --file-prefix|-fp PREFIX where\n"                              ..
        "     PREFIX : Prefix of the file to write results to\n"           ..
        "     NOTE   : .txt is added to the PREFIX so\n"                   ..
        "              -fp eg will write to eg.txt\n"                      ..
        "              By default a single file is written, (-wm 2)\n"     ..
        "              SEE --write-mode for more options\n\n"              ..
        "   --write-mode|-wm MODE where\n"                                 ..
        "     MODE : Writing mode for stats, options:\n"                   ..
        "       0  : Don't write or show stats\n"                          ..
        "       1  : Print stats to console (default)\n"                   ..
        "       2  : Write all core stats to a global file\n"              ..
        "       3  : Write separate stats for each core\n\n"               ..
        "   --src-mac|-sm SRC_MAC where\n"                                 ..
        "     SRC_MAC : Base SRC mac address (aa:bb:cc:dd:ee:ff)\n\n"      ..
        "   --dst-mac|-dm DST_MAC where\n"                                 ..
        "     DST_MAC : Base DST mac address (aa:bb:cc:dd:ee:ff)\n\n"      ..
        "   --src-ip|-sip SRC_IP where\n"                                  ..
        "     SRC_IP : Base SRC ip address (A.B.C.D)\n\n"                  ..
        "   --dst-ip|-dip DST_IP where\n"                                 ..
        "     DST_IP : Base DST ip address (A.B.C.D)\n\n"                  ..
        "   --src-port|-spt SRC_PORT where\n"                              ..
        "     SRC_PORT : Base source UDP port\n\n"                         ..
        "   --dst-port|-dpt DST_PORT where\n"                              ..
        "     DST_PORT : Base destination UDP port\n\n"                    ..
        "   --src-mac-vary|-smv SRC_MAC_VARY where\n"                      ..
        "     SRC_MAC_VARY : Variation in SRC mac address between flows\n" ..
        "                    (aa:bb:cc:dd:ee:ff)\n\n"                      ..
        "   --dst-mac-vary|-dmv DST_MAC_VARY where\n"                      ..
        "     DST_MAC_VARY : Variation in DST mac address between flows\n" ..
        "                    (aa:bb:cc:dd:ee:ff)\n\n"                      ..
        "   --src-ip-vary|-sipv SRC_IP_VARY where\n"                       ..
        "     SRC_IP_VARY : Variation in SRC ip address between flows\n"   ..
        "                   (A.B.C.D)\n\n"                                 ..
        "   --dst-ip-vary|-dipv DST_IP_VARY where\n"                       ..
        "     DST_IP_VARY : Variation in DST ip address between flows\n"   ..
        "                   (A.B.C.D)\n\n"                                 ..
        "   --src-port-vary|-sptv SRC_PORT_VARY where\n"                   ..
        "     SRC_PORT_VARY : Variation in SRC port between flows\n\n"     ..
        "   --dst-port-vary|-dptv DST_PORT_VARY where\n"                   ..
        "     DST_PORT_VARY : Variation in DST port between flows\n\n"     ..
        "   --src-mac-stride|-sms SRC_MAC_STRIDE where\n"                  ..
        "     SRC_MAC_STRIDE : Variation in SRC mac address between\n"     ..
        "                      streams (aa:bb:cc:dd:ee:ff)\n\n"            ..
        "   --dst-mac-stride|-dms DST_MAC_STRIDE where\n"                  ..
        "     DST_MAC_STRIDE : Variation in DST mac address between\n"     ..
        "                      streams (aa:bb:cc:dd:ee:ff)\n\n"            ..
        "   --src-ip-stride|-sips SRC_IP_STRIDE where\n"                   ..
        "     SRC_IP_STRIDE : Variation in SRC ip address between\n"       ..
        "                     streams (A.B.C.D)\n\n"                       ..
        "   --dst-ip-stride|-dips DST_IP_STRIDE where\n"                   ..
        "     DST_IP_STRIDE: Variation in DST ip address between\n"        ..
        "                     streams (A.B.C.D)\n\n"                       ..
        "   --src-port-stride|-spts SRC_PORT_STRIDE where\n"               ..
        "     SRC_PORT_STRIDE : Variation in SRC port between streams\n\n" ..
        "   --dst-port-stride|-dpts DST_PORT_STRIDE where\n"               ..
        "     DST_PORT_STRIDE : Variation in DST port between streams\n\n" ..
        "   --help|-h Print usage\n\n"                                     ..
        "   NOTE: Later arguments will overwrite earlier ones.\n"
    )
    print(usage)
end
