Browse Source

New upstream version 1.7.0

tags/upstream/1.7.0^0
Bernd Zeimetz 2 years ago
parent
commit
31bbd31e1a
100 changed files with 7064 additions and 3498 deletions
  1. 5
    3
      AUTHORS
  2. 439
    374
      CONFIG-KEYS
  3. 2
    2
      COPYING
  4. 317
    2
      ChangeLog
  5. 143
    118
      FAQS
  6. 2
    2
      INSTALL
  7. 10
    3
      Makefile.in
  8. 526
    289
      QUICKSTART
  9. 13
    11
      TOOLS
  10. 81
    7
      UPGRADE
  11. 1
    0
      aclocal.m4
  12. 4
    0
      bin/configure-help-replace.txt
  13. 742
    82
      configure
  14. 223
    22
      configure.ac
  15. 19
    18
      docs/INTERNALS
  16. 200
    83
      docs/MSGLOG_DUMP_FORMATS
  17. 12
    9
      docs/SIGNALS
  18. 9
    10
      docs/TRIGGER_VARS
  19. 15
    6
      examples/agent_to_peer.map.example
  20. 139
    15
      examples/amqp/amqp_receiver.py
  21. 76
    0
      examples/avro/avro_file_decoder.py
  22. 3
    0
      examples/flow_to_rd.map.example
  23. 183
    0
      examples/kafka/kafka_confluent_consumer.py
  24. 133
    20
      examples/kafka/kafka_consumer.py
  25. 2
    2
      examples/networks.lst.example
  26. 34
    31
      examples/pretag.map.example
  27. 6
    5
      examples/primitives.lst.example
  28. 3
    0
      examples/tee_receivers.lst.example
  29. 0
    76
      m4/ac_check_typedef.m4
  30. 84
    0
      m4/ax_lib_mysql.m4
  31. 10
    2
      sql/README.GeoIP
  32. 11
    0
      sql/README.IPv6
  33. 61
    30
      sql/README.mysql
  34. 23
    7
      sql/README.pgsql
  35. 44
    23
      sql/README.sqlite3
  36. 28
    0
      sql/README.tunnel
  37. 55
    5
      src/Makefile.am
  38. 183
    78
      src/Makefile.in
  39. 45
    9
      src/acct.c
  40. 43
    4
      src/addr.c
  41. 5
    2
      src/addr.h
  42. 47
    180
      src/amqp_common.c
  43. 4
    3
      src/amqp_common.h
  44. 200
    103
      src/amqp_plugin.c
  45. 3
    18
      src/amqp_plugin.h
  46. 5
    4
      src/bgp/Makefile.am
  47. 24
    8
      src/bgp/Makefile.in
  48. 349
    17
      src/bgp/bgp.c
  49. 38
    3
      src/bgp/bgp.h
  50. 0
    152
      src/bgp/bgp_community.c
  51. 0
    2
      src/bgp/bgp_community.h
  52. 4
    250
      src/bgp/bgp_ecommunity.c
  53. 0
    2
      src/bgp/bgp_ecommunity.h
  54. 304
    0
      src/bgp/bgp_lcommunity.c
  55. 69
    0
      src/bgp/bgp_lcommunity.h
  56. 104
    129
      src/bgp/bgp_logdump.c
  57. 1
    1
      src/bgp/bgp_logdump.h
  58. 74
    35
      src/bgp/bgp_lookup.c
  59. 3
    3
      src/bgp/bgp_lookup.h
  60. 371
    189
      src/bgp/bgp_msg.c
  61. 10
    6
      src/bgp/bgp_msg.h
  62. 33
    1
      src/bgp/bgp_packet.h
  63. 7
    5
      src/bgp/bgp_table.c
  64. 10
    19
      src/bgp/bgp_table.h
  65. 257
    43
      src/bgp/bgp_util.c
  66. 18
    6
      src/bgp/bgp_util.h
  67. 10
    3
      src/bmp/Makefile.in
  68. 7
    8
      src/bmp/bmp.c
  69. 9
    2
      src/bmp/bmp.h
  70. 80
    152
      src/bmp/bmp_logdump.c
  71. 6
    5
      src/bmp/bmp_lookup.c
  72. 2
    2
      src/bmp/bmp_lookup.h
  73. 54
    14
      src/bmp/bmp_msg.c
  74. 4
    2
      src/bmp/bmp_msg.h
  75. 87
    45
      src/bmp/bmp_util.c
  76. 10
    6
      src/bmp/bmp_util.h
  77. 2
    3
      src/bpf_filter.c
  78. 35
    16
      src/cfg.c
  79. 42
    24
      src/cfg.h
  80. 628
    465
      src/cfg_handlers.c
  81. 40
    23
      src/cfg_handlers.h
  82. 31
    54
      src/classifier.c
  83. 1
    3
      src/classifier.h
  84. 16
    16
      src/conntrack.c
  85. 66
    84
      src/imt_plugin.c
  86. 5
    3
      src/imt_plugin.h
  87. 11
    11
      src/ip_flow.c
  88. 3
    3
      src/ip_flow.h
  89. 20
    11
      src/ip_frag.c
  90. 10
    3
      src/isis/Makefile.in
  91. 4
    3
      src/isis/isis.c
  92. 1
    3
      src/isis/isis.h
  93. 1
    0
      src/isis/isis_circuit.h
  94. 1
    0
      src/isis/isis_lsp.c
  95. 2
    0
      src/isis/isis_lsp.h
  96. 2
    0
      src/isis/isis_pdu.c
  97. 0
    1
      src/isis/isisd.c
  98. 1
    0
      src/isis/isisd.h
  99. 4
    4
      src/isis/iso.h
  100. 0
    0
      src/isis/iso_checksum.c

+ 5
- 3
AUTHORS View File

@@ -1,5 +1,5 @@
pmacct (Promiscuous mode IP Accounting package) v1.6.1
pmacct is Copyright (C) 2003-2016 by Paolo Lucente
pmacct [IP traffic accounting : BGP : BMP : IGP : Streaming Telemetry]
pmacct is Copyright (C) 2003-2017 by Paolo Lucente

Founder:

@@ -24,9 +24,9 @@ Thanks to the following people for their strong support along the time:
Robert Blechinger
Stefano Birmani
Codethink.co.uk
Pier Carlo Chiodi
Arnaud De-Bermingham
Francois Deppierraz
Marcello Di Leonardo
Pierre Francois
Rich Gade
Aaron Glenn
@@ -46,7 +46,9 @@ Thanks to the following people for their strong support along the time:
Gabriel Snook
Rene Stoutjesdijk
Thomas Telkamp
Matthieu Texier
Stig Thormodsrud
Luca Tosolini
Brent Van Dussen
Markus Weber
Chris Wilson

+ 439
- 374
CONFIG-KEYS
File diff suppressed because it is too large
View File


+ 2
- 2
COPYING View File

@@ -1,5 +1,5 @@
pmacct (Promiscuous mode IP Accounting package)
pmacct is Copyright (C) 2003-2016 by Paolo Lucente
pmacct [IP traffic accounting : BGP : BMP : IGP : Streaming Telemetry]
pmacct is Copyright (C) 2003-2017 by Paolo Lucente


GNU GENERAL PUBLIC LICENSE

+ 317
- 2
ChangeLog View File

@@ -1,5 +1,320 @@
pmacct (Promiscuous mode IP Accounting package) v1.6.1
pmacct is Copyright (C) 2003-2016 by Paolo Lucente
pmacct [IP traffic accounting : BGP : BMP : IGP : Streaming Telemetry]
pmacct is Copyright (C) 2003-2017 by Paolo Lucente

The keys used are:
!: fixed/modified feature, -: deleted feature, +: new feature

1.7.0 -- 21-10-2017
+ ZeroMQ integration: by defining plugin_pipe_zmq to 'true', ZeroMQ is
used for queueing between the Core Process and plugins. This is in
alternative to the home-grown circular queue implementation (ie.
plugin_pipe_size). plugin_pipe_zmq_profile can be set to one value
of { micro, small, medium, large, xlarge } and allows to select
among a few standard buffering profiles without having to fiddle
with plugin_buffer_size. How to compile, install and operate ZeroMQ
is documented in the "Internal buffering and queueing" section of
the QUICKSTART document.
+ nDPI integration: enables packet classification, replacing existing
L7-layer project integration, and is available for pmacctd and
uacctd. The feature, once nDPI is compiled in, is simply enabled by
specifying 'class' as part of the aggregation method. How to compile
install and operate nDPI is documented in the "Quickstart guide to
packet classification" section of the QUICKSTART document.
+ nfacctd: introduced nfacctd_templates_file so that NetFlow v9/IPFIX
templates can be cached to disk to limit the amount of lost packets
due to unknown templates when nfacctd (re)starts. The implementation
is courtesy by Codethink Ltd.
+ nfacctd: introduced support for PEN on IPFIX option templates. This
is in addition to already supported PEN for data templates. Thanks
to Gilad Zamoshinski ( @zamog ) for his support.
+ sfacctd: introduced new aggregation primitives (tunnel_src_host,
tunnel_dst_host, tunnel_proto, tunnel_tos) to support inner L3
layers. Thanks to Kaname Nishizuka ( @__kaname__ ) for his support.
+ nfacctd, sfacctd: pcap_savefile and pcap_savefile_wait were ported
from pmacctd. They allow to process NetFlow/IPFIX and sFlow data
from previously captured packets; these also ease some debugging by
not having to resort anymore to tcpreplay for most cases.
+ pmacctd, sfacctd: nfacctd_time_new feature has been ported so, when
historical accounting is enabled, to allow to choose among capture
time and time of receipt at the collector for time-binning.
+ nfacctd: added support for NetFlow v9/IPFIX field types #130/#131,
respectively the IPv4/IPv6 address of the element exporter.
+ nfacctd: introduced nfacctd_disable_opt_scope_check: mainly a work
around to implementations not encoding NetFlow v9/IPIFX option scope
correctly, this knob allows to disable option scope checking. Thanks
to Gilad Zamoshinski ( @zamog ) for his support.
+ pre_tag_map: added 'source_id' key for tagging on NetFlow v9/IPFIX
source_id field. Added also 'fwdstatus' for tagging on NetFlow v9/
IPFIX information element #89: this implementation is courtesy by
Emil Palm ( @mrevilme ).
+ tee plugin: tagging is now possible on NetFlow v5-v8 engine_type/
engine_id, NetFlow v9/IPFIX source_id and sFlow AgentId.
+ tee plugin: added support for 'src_port' in tee_receivers map. When
in non-transparent replication mode, use the specified UDP port to
send data to receiver(s). This is in addition to tee_source_ip,
which allows to set a configured IP address as source.
+ networks_no_mask_if_zero: a new knob so that IP prefixes with zero
mask - that is, unknown ones or those hitting a default route - are
not masked. The feature applies to *_net aggregation primitives and
makes sure individual IP addresses belonging to unknown IP prefixes
are not zeroed out.
+ networks_file: hooked up networks_file_no_lpm feature to peer and
origin ASNs and (BGP) next-hop fields.
+ pmacctd: added support for calling pcap_set_protocol() if supported
by libpcap. Patch is courtesy by Lennert Buytenhek ( @buytenh ).
+ pmbgpd, pmbmpd, pmtelemetryd: added a few CL options to ease output
of BGP, BMP and Streaming Telemetry data, for example: -o supplies
a b[gm]p_daemon_msglog_file, -O supplies a b[gm]p_dump_file and -i
supplies b[gm]p_dump_refresh_time.
+ kafka plugin: in the examples section, added a Kafka consumer script
using the performing confluent-kafka-python module.
! fix, BGP daemon: segfault with add-path enabled peers as per issue
#128. Patch is courtesy by Markus Weber ( @FvDxxx ).
! fix, print plugin: do not update link to latest file if cause of
purging is a safe action (ie. cache space is finished. Thanks to
Camilo Cardona ( @jccardonar ) for reporting the issue. Also, for
the same reason, do not execute triggers (ie. print_trigger_exec).
! fix, nfacctd: improved IP protocol check in NF_evaluate_flow_type()
A missing length check was causing, under certain conditions, some
flows to be marked as IPv6. Many thanks to Yann Belin for his
support resolving the issue.
! fix, print and SQL plugins: optimized the cases when the dynamic
filename/table has to be re-evaluated. This results in purge speed
gains when the dynamic part is time-related and nfacctd_time_new is
set to true.
! fix, bgp_daemon_md5_file: if the server socket is AF_INET and the
compared peer address in MD5 file is AF_INET6 (v4-mapped v6), pass
it through ipv4_mapped_to_ipv4(). Also if the server socket is
AF_INET6 and the compared peer addess in MD5 file is AF_INET, pass
it through ipv4_to_ipv4_mapped(). Thanks to Paul Mabey for reporting
the issue.
! fix, nfacctd: improved length checks in resolve_vlen_template() to
prevent SEGVs. Thanks to Josh Suhr and Levi Mason for their support.
! fix, nfacctd: flow stitching, improved flow end time checks. Thanks
to Fabio Bindi ( @FabioLiv ) for his support resolving the issue.
! fix, amqp_common.c: amqp_persistent_msg now declares the RabbitMQ
exchange as durable in addition to marking messages as persistent;
this is related to issue #148.
! fix, nfacctd: added flowset count check to existing length checks
for NetFlow v9/IPFIX datagrams. This is to avoid logs flooding in
case of padding. Thanks to Steffen Plotner for reporting the issue.
! fix, BGP daemon: when dumping BGP data at regular time intervals,
dump_close message contained wrongly formatted timestamp. Thanks to
Yuri Lachin for reporting the issue.
! fix, MySQL plugin: if --enable-ipv6 and sql_num_hosts set to true,
use INET6_ATON for both v4 and v6 addresses. Thanks to Guy Lowe
( @gunkaaa ) for reporting the issue and his support resolving it.
! fix, 'flows' primitive: it has been wired to sFlow so to count Flow
Samples received. This is to support Q21 in FAQS document.
! fix, BGP daemon: Extended Communities value was printed with %d
(signed) format string instead of %u (unsigned), causing issue on
large values.
! fix, aggregate_primitives: improved support of 'u_int' semantics for
8 bytes integers. This is in addition to already supported 1, 2 and
4 bytes integers.
! fix, pidfile: pidfile created by plugin processes was not removed.
Thanks to Yuri Lachin for reporting the issue.
! fix, print plugin: checking non-null file descriptor before setvbuf
in order to prevent SEGV. Similar checks were added to prevent nulls
be input to libavro calls when Apache Avro output is selected.
! fix, SQL plugins: MPLS aggregation primitives were not correctly
activated in case sql_optimize_clauses was set to false.
! fix, building system: reviewed minimum requirement for libraries,
removed unused m4 macros, split features in plugins (ie. MySQL) and
supports (ie. JSON).
! fix, sql_history: it now correctly honors periods expressed is 's'
seconds.
! fix, BGP daemon: rewritten bgp_peer_print() to be thread safe.
! fix, pretag.h: addressed compiler warning on 32-bit architectures,
integer constant is too large for "long" type. Thanks to Stephen
Clark ( @sclark46 ) for reporting the issue.
- MongoDB plugin: it is being discontinued since the old Mongo API is
not supported anymore and there has never been enough push from the
community to transition to the new/current API (which would require
a rewrite of most of the plugin). In this phase-1 the existing
MongoDB plugin is still available using 'plugins: mongodb_legacy'
in the configuration.
- Packet classification basing on the L7-filter project is being
discontinued (ie. 'classifiers' directive). This is being replaced
by an implementation basing on the nDPI project. As part of this
also the sql_aggressive_classification knob has been discontinued.
- tee_receiver was part of the original implementation of the tee
plugin, allowing to forward to a single target and hence requiring
multiple plugins instantiated, one per target. Since 0.14.3 this
directive was effectively outdated by tee_receivers.
- tmp_net_own_field: the knob has been discontinued and was allowing
to revert to backward compatible behaviour of IP prefixes (ie.
src_net) being written in the same field as IP addresses (ie.
src_host).
- tmp_comms_same_field: the knob has been discontinued and was
allowing to revert to backward compatible behaviour of BGP
communities (standard and extended) being writeen all in the same
field.
- plugin_pipe_amqp and plugin_pipe_kafka features were meant as an
alternative to the homegrown queue solution for internal messaging,
ie. passing data from the Core Process to Plugins, and are being
discontinued. They are being replaced by a new implementation,
plugin_pipe_zmq, basing on ZeroMQ.
- plugin_pipe_backlog was allowing to keep an artificial backlog of
data in the Core Process so to maximise bypass poll() syscalls in
plugins. If home-grown queueing is found limiting, instead of
falling back to such strategies, ZeroMQ queueing should be used.
- pmacctd: deprecated support for legacy link layers: FDDI, Token Ring
and HDLC.

1.6.2 -- 21-04-2017
+ BGP, BMP daemons: introduced support for BGP Large Communities IETF
draft (draft-ietf-idr-large-community). Large Communities are stored
in a variable-length field. Thanks to Job Snijders ( @job ) for his
support.
+ BGP daemon: implemented draft-ietf-idr-shutdown. The draft defines a
mechanism to transmit a short freeform UTF-8 message as part of a
Cease NOTIFICATION message to inform the peer why the BGP session is
being shutdown or reset. Thanks to Job Snijders ( @job ) for his
support.
+ tee plugin, pre_tag_map: introduced support for inspetion of specific
flow primitives and selective replication over them. The primitives
supported are: input and output interfaces, source and destination
MAC addresses, VLAN ID. The feature is now limited to sFlow v5 only.
Thanks to Nick Hilliard and Barry O'Donovan for their support.
+ Added src_host_pocode and dst_host_pocode primitives, pocode being a
compact and (de-)aggregatable (easy to identify districts, cities,
metro areas, etc.) geographical representation, based on the Maxmind
v2 City Database. Thanks to Jerred Horsman for his support.
+ Kafka support: introduced support for user-defined (librdkafka) config
file via the new *_kafka_config_file config directives. Full pathname
to a file containing directives to configure librdkafka is expected.
All knobs whose values are string, integer, boolean are supported.
+ AMQP, Kafka plugins: introduced new directives kafka_avro_schema_topic,
amqp_avro_schema_routing_key to transmit Apache Avro schemas at regular
time intervals. The routing key/topic can overlap with the one used to
send actual data.
+ AMQP, Kafka plugins: introduced support for start/stop markers when
encoding is set to Avro (ie. 'kafka_output: avro'); also Avro schema
is now embedded in a JSON envelope when sending it via a topic/routing
key (ie. kafka_avro_schema_topic).
+ print plugin: introduced new config directive avro_schema_output_file
to save the Apache Avro schema in a separate file (it was only possible
to have it combined at the beginning of the data file).
+ BGP daemon: introduced a new bgp_daemon_as config directive to set a
LocalAS which could be different from the remote peer one. This is to
establish an eBGP session instead of a iBGP one (default).
+ flow_to_rd_map: introduced support for mpls_vpn_id. In NetFlow/IPFIX
this is compared against Field Types #234 and #235.
+ sfacctd: introduced support for sFlow v2/v4 counter samples (generic,
ethernet, vlan). This is in addition to existing support for sFlow v5
counters.
+ BGP, BMP and Streming Telemetry daemons: added writer_id field when
writing to Kafka and/or RabbitMQ. The field reports the configured
core_proc_name and the actual PID of the writer process (so, while
being able to correlate writes to the same daemon, it's also possible
to distinguish among overlapping writes).
+ amqp, kafka, print plugins: harmonized JSON output to the above: added
event_type field, writer_id field with plugin name and PID.
+ BGP, BMP daemons: added AFI, SAFI information to log and dump outputs;
also show VPN Label if SAFI is MPLS VPN.
+ pmbgpd, pmbmpd: added logics to bypass building RIBs if only logging
BGP/BMP data real-time.
+ BMP daemon: added BMP peer TCP port to log and dump outputs (for NAT
traversal scenarios). Contextually, multiple TCP sessions per IP are
now supported for the same reason.
+ SQL plugins: ported (from print, etc. plugins) the 1.6.1 re-working of
the max_writers feature.
+ uacctd: use current time when we don't have a timestamp from netlink.
We only get a timestamp when there is a timestamp in the skb. Notably,
locally generated packets don't get a timestamp. The patch is courtesy
by Vincent Bernat ( @vincentbernat ).
+ build system: added configure options for partial linking of binaries
with any selection/combination of IPv4/IPv6 accounting daemons, BGP
daemon, BMP daemon and Streaming Telemetry daemon possible. By default
all are compiled in.
+ BMP daemon: internal code changes to pass additional info from BMP
per-peer header to bgp_parse_update_msg(). Goal is to expose further
info, ie. pre- vs post- policy, when logging or dumping BMP info.
! fix, BGP daemon: introduced parsing of IPv6 MPLS VPN (vpnv6) NLRIs.
Thanks to Alberto Santos ( @m4ccbr ) for reporting the issue.
! fix, BGP daemon: upon doing routes lookup, now correctly honouring
the case of BGP-LU (SAFI_MPLS_LABEL).
! fix, BGP daemon: send BGP NOTIFICATION out in case of known failures
in bgp_parse_msg().
! fix, kafka_partition, *_kafka_partition: default value changed from 0
(partition zero) to -1 (RD_KAFKA_PARTITION_UA, partition unassigned).
Thanks to Johan van den Dorpe ( @johanek ) for his support.
! fix, pre_tag_map: removed constraint for 'ip' keyword for nfacctd and
sfacctd maps. While this is equivalent syntax to specifying rules with
'ip=0.0.0.0/0', it allows for map indexing (maps_index: true).
! fix, bgp_agent_map: improved sanity check against bgp_ip for IPv6
addresses (ie. an issue appeared for the case of '::1' where the first
64 bits are zeroed out). Thanks to Charlie Smurthwaite ( @catphish )
for reporting the issue.
! fix, maps_index: indexing now correctly works for IPv6 pre_tag_map
entries. That is, those where 'ip', the IP address of the NetFlow/
IPFIX/sFlow exporter, is an IPv6 address.
! fix, pre_tag_map: if mpls_vpn_rd matching condition is specified and
maps_index is enabled, PT_map_index_fdata_mpls_vpn_rd_handler() now
picks the right (and expected) info.
! fix, pkt_handlers.c: improved definition and condition to free() in
bgp_ext_handler() in order to prevent SEGVs. Thanks to Paul Mabey for
his support.
! fix, kafka_common.c: removed waiting time from p_kafka_set_topic().
Added docs advicing to create in advance Kafka topics.
! fix, sfacctd, sfprobe: tag and tag2 are now correctly re-defined as
64 bits long.
! fix, sfprobe plugin, sfacctd: tags and class primitives are now being
encoded/decoded using enterprise #43874, legit, instead of #8800, that
was squatted back in the times. See issue #71 on GiHub for more info.
! fix, sfacctd: lengthCheck() + skipBytes() were producing an incorrect
jump in case of unknown flow samples. Replaced by skipBytesAndCheck().
Thanks to Elisa Jasinska ( @fooelisa ) for her support.
! fix, pretag_handlers.c: in bgp_agent_map added case for 'vlan and ...'
filter values.
! fix, BGP daemon: multiple issues of partial visibility of the stored
RIBs and SEGVs when bgp_table_per_peer_buckets was not left default:
don't mess with bms->table_per_peer_buckets given the multi-threaded
scenario. Thanks to Dan Berger ( @dfberger ) for his support.
! fix, BGP, BMP daemons: bgp_process_withdraw() function init aligned to
bgp_process_update() in order to prevent SEGVs. Thanks to Yuri Lachin
for his support.
! fix, bgp_msg.c: Route Distinguisher was stored and printed incorrectly
when of type RD_TYPE_IP. Thanks to Alberto Santos ( @m4ccbr ) for
reporting the issue.
! fix, bgp_logdump.c: p_kafka_set_topic() was being wrongly applied to
an amqp_host structure (instead of a kafka_host structure). Thanks to
Corentin Neau ( @weyfonk ) for reporting the issue.
! fix, BGP daemon: improved BGP next-hop setting and comparison in cases
of MP_REACH_NLRI and MPLS VPNs. Many thanks to both Catalin Petrescu
( @cpmarvin ) and Alberto Santos ( @m4ccbr ) for their support.
! fix, pmbgpd, pmbmpd: pidfile was not written even if configured. Thanks
to Aaron Glenn ( @aaglenn ) for reporting the issue.
! fix, tee plugin: tee_max_receiver_pools is now correctly honoured and
debug message shows the replicatd protocol, ie. NetFlow/IPFIX vs sFlow.
! AMQP, Kafka plugins: separate JSON objects, newline separated, are
preferred to JSON arrays when buffering of output is enabled (ie.
kafka_multi_values) and output is set to JSON. This is due to quicker
serialisation performance shown by the Jansson library.
! build system: switched to enable IPv6 support by default (while the
--disable-ipv6 knob can be used to reverse the behaviour). Patch is
courtesy by Elisa Jasinska ( @fooelisa ).
! build system: given visibility, ie. via -V CL option, into compile
options enabled by default (ie. IPv6, threads, 64bit counters, etc.).
! fix, nfprobe: free expired records when exporting to an unavailable
collector in order to prevent a memory leak. Patch is courtersy by
Vladimir Kunschikov ( @kunschikov ).
! fix, AMQP plugin: set content type to binary in case of Apache Avro
output.
! fix, AMQP, Kafka plugins: optimized amqp_avro_schema_routing_key and
kafka_avro_schema_topic. Avro schema is built only once at startup.
! fix, cfg.c: improved parsing of config key-values where squared brakets
appear in the value part. Thanks to Brad Hein ( @regulatre ) for
reporting the issue. Also, detection of duplicates among plugin and
core process names was improved.
! fix, misc: compiler warnings: fix up missing includes and prototypes;
the patch is courtesy by Tim LaBerge ( @tlaberge ).
! kafka_consumer.py, amqp_receiver.py: Kafka, RabbitMQ consumer example
scripts have been greatly expanded to support posting to a REST API or
to a new Kafka topic, including some stats. Also conversion of multiple
newline-separated JSON objects to a JSON array has been added. Misc
bugs were fixed.

1.6.1 -- 31-10-2016
+ Introduced pmbgpd daemon: a stand-alone BGP collector daemon; acts as a

+ 143
- 118
FAQS View File

@@ -1,8 +1,9 @@
pmacct (Promiscuous mode IP Accounting package)
pmacct is Copyright (C) 2003-2016 by Paolo Lucente
pmacct [IP traffic accounting : BGP : BMP : IGP : Streaming Telemetry]
pmacct is Copyright (C) 2003-2017 by Paolo Lucente

Q1: What is pmacct project homepage ?
A: pmacct homepage is http://www.pmacct.net/
A: pmacct homepage is http://www.pmacct.net/ . pmacct is also present on GitHub at
the URL: https://github.com/pmacct/pmacct .


Q2: 'pmacct', 'pmacctd', 'nfacctd', 'sfacctd', 'uacctd', 'pmtelemetryd',
@@ -12,62 +13,63 @@ A: 'pmacct' is intended to be the name of the project; 'pmacctd' is the name of
(versions supported NetFlow v1 to v9) and IPFIX accounting daemon; 'sfacctd' is
the name of the sFlow v2/v4/v5 accounting daemon; 'uacctd' is the name of the
Linux Netlink NFLOG-based accounting daemon (historically, it was using ULOG,
hence its name); 'pmtelemetryd' is the name of the streaming network telemetry
collector daemon, where quoting Cisco IOS-XR Telemetry Configuration Guide at the
time of this writing "Streaming telemetry [ .. ] data can be used for analysis
and troubleshooting purposes to maintain the health of the network. This is
achieved by leveraging the capabilities of machine-to-machine communication.
[ .. ]"; 'pmbgpd' is the name of the pmacct BGP collector daemon; 'pmbmpd' is
the name of the pmacct BMP collector daemon.
hence its name); 'pmtelemetryd' is the name of the Streaming Telemetry collector
daemon, where, quoting Cisco IOS-XR Telemetry Configuration Guide at the time of
this writing, "Streaming telemetry [ .. ] data can be used for analysis and
troubleshooting purposes to maintain the health of the network. This is achieved
by leveraging the capabilities of machine-to-machine communication. [ .. ]";
'pmbgpd' is the name of the pmacct BGP collector daemon; 'pmbmpd' is the name of
the pmacct BMP collector daemon.


Q3: Does pmacct stand for Promiscuous mode IP Accounting package ?
A: That is not entirely correct today, it was originally though. pmacct born as a
libpcap-based project only. Over the time it evolved to include NetFlow first,
sFlow shortly afterwards and NFLOG more recently - this is striving to maintain
a consistent implementation over the set, unless technical considerations
prevent that to happen for specific cases.
Q3: Does pmacct stand for Promiscuous mode IP Accounting package?
A: Not anymore, it was originally though: pmacct was born as a libpcap-based traffic
collection project only. Over the time it evolved to include NetFlow first, sFlow
shortly afterwards and NFLOG more recently. Latest additions being in the areas
of BGP, BMP and Streaming Telemetry. However the unpronounceable name 'pmacct'
remained as a distinctive signature of the project.


Q4: What are pmacct main features?
A: pmacct can collect, replicate and export network data. Collect in memory tables,
store persistently to RDBMS (MySQL, PostgreSQL, SQLite 3.x), noSQL databases
(key-value: BerkeleyDB 5.x via SQLite API or document-oriented: MongoDB) and
flat-files (csv, formatted, JSON, Apache Avro output), publish to AMQP and
Kafka brokers (ie. to insert in ElasticSearch, Cassandra or CouchDB). Export
speaking sFlow v5, NetFlow v1/v5/v9 and IPFIX. pmacct is able to perform data
aggregation, offering a rich set of primitives to choose from; it can also
filter, sample, renormalize, tag and classify at L7. pmacct integrates a BGP
daemon join routing visibility and network traffic information.


Q5: Does any of the pmacct daemons logs to flat files?
A: Yes, but in a specific way. In other tools flat-files are typically used to log every
micro-flow (or whatever aggregation the NetFlow agents have been configured to export
with) and work in a two-stages fashion: a) write down to persistent storage then b)
consolidate, on either or both spatial and temporal axes, to build the desired view.
By inception, pmacct always aimed to a single-stage approach instead, ie. offer data
reduction tecniques and correlation tools to process network traffic data on the fly,
so to immediately offer the desired view(s) of the traffic. pmacct writes to files in
text-format (json, csv or formatted via its 'print' plugin, see QUICKSTART doc for
further information) so to maximize potential integration with 3rd party applications
while keeping low the effort of customization.


Q6: Is it feasible for pmacct to scale by making use of either memory tables or RDBMS
as backend for logging network traffic?
A: pmacct was not originally meant to log network traffic at packet/micro-flow level: it
allows to get an aggregated view of the traffic -- both in space and in time. On top
of that, there are layers of filtering, sampling and tagging. These are the keys to
scale. As these features are fully configurable, data granularity and resolution can
be traded off in favour of increased scalability or less resources consumption. More
recently, logging has been introduced, by means of two new primitives timestamp_start
and timestamp_end, fostered by the development of NetFlow/IPFIX as generic transport
protocol, ie. as a replacement of syslog; it was then intuitive to generalize the
logging support to the more traditional traffic accounting part.


Q7: I see my daemon taking much CPU cycles; is there a way to reduce the load?
A: pmacct can collect, replicate and export network information. On the data plane
(ie. IPv4/IPv6 traffic) it can cache in memory tables, store persistently to
RDBMS (MySQL, PostgreSQL, SQLite 3.x), noSQL databases (key-value: BerkeleyDB
5.x via SQLite API or document-oriented: MongoDB) and flat-files (CSV, formatted,
JSON, Apache Avro output), publish to AMQP and Kafka brokers (ie. to insert in
ElasticSearch, InfluxDB or Cassandra). Export speaking sFlow v5, NetFlow v1/v5/v9
and IPFIX. pmacct is able to perform data aggregation, offering a rich set of
primitives to choose from; it can also filter, sample, renormalize, tag and
classify at L7. On the control and infrastructure planes it can collect and
publish to AMQP and Kafka brokers BGP, BMP, IGP and Streaming Telemetry data
both standalone and as correlation/enrichment of data plane information.


Q5: Do pmacct IPv4/IPv6 traffic accounting daemons log to flat files?
A: Yes. But while in other tools flat-files are typically used to log every micro-flow
(or whatever aggregation the NetFlow agents have been configured to export with) and
work in a two-stages fashion, ie. a) write down to persistent storage then b)
consolidate to build the desired view, by inception, pmacct always aimed to a
single-stage approach instead, ie. offer data reduction tecniques and correlation
tools to process network traffic data on the fly, so to offer immediate view(s) of
the traffic. pmacct writes to files in text-format (JSON, Avro, CSV or formatted via
'print' plugin, and JSON or Avro via Kafka and AMQP plugins, see QUICKSTART doc for
further information) so to maximize integration with 3rd party tools while keeping
low the effort of customization.


Q6: What are the options to scale a pmacct deployment to match input data rate?
A: There are two dimensions to it: 1) scale within the same instance of pmacct: make use
of data reduction tecniques part of pmacct, ie. spatial and temporal aggregation,
filtering, sampling and tagging. As these features are fully configurable, going from
full micro-flow visibility to - say - node-to-node IP network traffic matrix, data
granularity/resolution can be traded off for scalability/resources consumption; 2)
divide-and-conquer input data over a set of pmacct instances by either balancing or
mapping data onto collectors. See next point, Q7, for libpcap; the 'tee' plugin can
be used for this purpose for NetFlow, IPFIX and sFlow.


Q7: I see my libpcap-based daemon (pmacctd) taking much CPU cycles; is there a way to
reduce the load?
A: CPU cycles are proportional to the amount of traffic (packets, flows, samples) that
the daemon receives; in case of pmacctd it's possible to reduce the CPU share by
avoiding unnecessary copies of data, also optimizing and buffering the necessary
@@ -93,17 +95,17 @@ A: CPU cycles are proportional to the amount of traffic (packets, flows, samples
Internal buffering can also help and, contrary to the previous tecniques, applies
to all daemons. Buffering is enabled with the plugin_buffer_size directive; buffers
can then be queued and distributed with a choice of an home-grown circolar queue
implementation (plugin_pipe_size) or a RabbitMQ broker (plugin_pipe_amqp). Check
CONFIG-KEYS and QUICKSTART for more information.
implementation (plugin_pipe_size) or a ZeroMQ queue (plugin_pipe_zmq). Check both
CONFIG-KEYS and QUICKSTART for more information.


Q8: I want to to account both inbound and outbound traffic of my network, with an host
breakdown; how to do that in a savy fashion ? Do i need to run two daemon instances
one per traffic direction ?
A: No, you will be able to leverage the pluggable architecture of the daemons: you will
run a single daemon with two plugins attached to it; each of these will get part of
the traffic (aggregate_filter), either outbound or inbound. A sample config snippet
follows:
breakdown; how to do that in a savy fashion? Do i need to run two daemon instances
one per traffic direction?
A: No, this is a toy case where you will be able to leverage the pluggable architecture
of the pmacct daemons: you will run a single daemon with two plugins attached to it;
each of these will get part of the traffic (aggregate_filter), either outbound or
inbound. A sample config snippet follows:

...
aggregate[inbound]: dst_host
@@ -134,11 +136,16 @@ A: No, you will be able to leverage the pluggable architecture of the daemons: y

Q9: I'm intimately fashioned by the idea of storing every single flow flying through my
network, before making up my mind what to do with such data: i basically would like
to aggregate my traffic as 'src_host, dst_host, src_port, dst_port, proto'. Is this
feasible without any filtering ?
A: This is not adviceable. A simple reason being this would result in a huge matrix of
data, whose behaviour and size would be totally un-predictable over time (ie. impact
of port scans, DDoS, etc.). Nevertless, it remains a valid configuration.
to (de-)aggregate my traffic as 'src_host, dst_host, src_port, dst_port, proto' or
'src_host, dst_host, src_port, dst_port, proto, timestamp_start, timestamp_end'. Is
this feasible without any filtering?
A: If such data granularity is required by the use-case addressed, ie. DDoS, forensics,
security, research, etc. then this can be achieved no problem with pmacct - you have
only to be careful planning for the right amount of system/cluster resources. In all
other cases this is not adviceable as this would result in a huge matrix of data -
meaning increased CPU, memory and disk usage - for no benefit - plus, to be always
considered, the impact of unexpected network events (ie. port scans, DDoS, etc.) on
the solution.


Q10: I use pmacctd. What portion of the packets is included into the bytes counter ?
@@ -157,17 +164,25 @@ A: The portion of the packet accounted starts from the IPv4/IPv6 header (inclusi
directly within pmacct via the 'adjb' action (sql_preprocess).


Q11: How to get the historical accounting enabled ? SQL table have a 'stamp_inserted'
and 'stamp_updated' fields but they remain empty.
A: Historical accounting is easily enabled by adding to the SQL plugin configuration a
'sql_history' directive. Associate to it a 'sql_history_roundoff'. For examples and
syntax, refer to CONFIG-KEYS and QUICKSTART documents.


Q12: CLI is not enough to me. I would like to graph traffic data: how to do that?
A: RRDtool, MRTG and GNUplot are just some tools which could be easily integrated with
pmacct operations. 'Memory plugin' is suitable as temporary storage and allows to
easily retrieve counters:
Q11: What is historical accounting feature and how to get it configured?
A: pmacct allows to optionally define arbitrary time-bins (ie. 5 mins, 1 hour, etc.)
and assign collected data to it basing on a timestamp. This is in brief called
historical accounting and is enabled via *history* directives (ie. print_history,
print_history_roundoff, sql_history, etc.). The time-bin to which data is allocated
to is stored in the 'stamp_inserted' field (if supported by the plugin in use, ie.
all except 'print', where to avoid redundancy this is encoded as part of the file
name, and 'memory'). Flow data is by default assigned to a time-bin basing on its
start time or - not applying that or missing such info - the timestamp of the whole
datagram or - not applying that or missing such info - the time of arrival at the
collector. Where multiple choices are supported, ie. NetFlow/IPFIX, the directive
nfacctd_time_new allows to explicitely select the time source.


Q12: Counters via CLI are good for (email, web) reporting but not enough. What are the
options to graph network data?
A: An option could be to use traditional graphing tools like RRDtool, MRTG and GNUplot
in conjunction with the 'memory' plugin. The plugin works as a cache and offers a
pull mechanism, the pmacct IMT client tool, that allows to easily retrieve counters:
shell> ./pmacctd -D -c src_host -P memory -i eth0
shell> ./pmacct -c src_host -N 192.168.4.133 -r
@@ -180,9 +195,8 @@ A: RRDtool, MRTG and GNUplot are just some tools which could be easily integrate

shell> rrdtool update 192_168_4_133.rrd N:`./pmacct -c src_host -N 192.168.4.133 -r`

Starting from 0.7.6, you will also be able to spawn as much as 4096 requests into a
single query; you may write your requests commandline (';' separated) but also read
them from a file (one per line):
Multiple requests can be batched as part of a single query, each request can be ';'
separated via CLI or read from an input file (one query per line):

shell> ./pmacct -c src_host,dst_host -N 192.168.4.133,192.168.0.101;192.168.4.5,192.168.4.1;... -r
50905
@@ -199,21 +213,10 @@ A: RRDtool, MRTG and GNUplot are just some tools which could be easily integrate
192.168.4.5,192.168.4.1
...

Furthermore, SNMP is a widespreaded protocol used (and widely supported) in the IP
accounting field to gather IP traffic information by network devices. 'pmacct' may
also be easily connected to Net-SNMP extensible MIB. What follows is an example for
your 'snmpd.conf':

exec .1.3.6.1.4.1.2021.50 Description /usr/local/bin/pmacct -c src_host -N 192.168.4.133 -r

Then, an 'snmpwalk' does the rest of the work:
shell> snmpwalk -v 1 localhost -c public .1.3.6.1.4.1.2021.50
.1.3.6.1.4.1.2021.50.1.1 = 1
.1.3.6.1.4.1.2021.50.2.1 = "Description"
.1.3.6.1.4.1.2021.50.3.1 = "/usr/local/bin/pmacct -c src_host -N 192.168.4.133 -r"
.1.3.6.1.4.1.2021.50.100.1 = 0
.1.3.6.1.4.1.2021.50.101.1 = "92984384"
.1.3.6.1.4.1.2021.50.102.1 = 0
A second option is to leverage one of the several modern data analytics stacks that
do typically comprise of data manipulation, storage and visualization. Pointers in
this sense would be the ELK stack (ElasticSearch, Logstash, Kibana) or the TICK
stack (Telegraf, InfluxDB, Chronograf, Kapacitor). Much more exist.


Q13: The network equipment i'm using supports sFlow but i don't know how to enable it.
@@ -224,8 +227,7 @@ A: If you are unable to enable sFlow commandline, you have to resort to the SNMP
available at the following URL: http://www.inmon.com/technology/sflowenable


Q14: I've configured the pmacct package in order to support IPv6 via the '--enable-ipv6'
switch. Now, when i launch either nfacctd or sfacctd i receive the following error
Q14: When i launch either nfacctd or sfacctd i receive the following error
message: ERROR ( default/core ): socket() failed. What to do ?
A: When IPv6 code is enabled, sfacctd and nfacctd will try to fire up an IPv6 socket.
The error message is very likely to be caused by the proper kernel module not being
@@ -251,9 +253,9 @@ A: pmacct tarball gets with so called 'default' tables (IP and BGP); they are bu

Q16: What is the best way to kill a running instance of pmacct avoiding data loss ?
A: Two ways. a) Simply kill a specific plugin that you don't need anymore: you will
have to identify it and use the 'kill -INT <process number> command; b) kill the
have to identify it and use the 'kill -INT <process number>' command; b) kill the
whole pmacct instance: you can either use the 'killall -INT <daemon name>' command
or identify the Core Process and use the 'kill -INT <process number> command. All
or identify the Core Process and use the 'kill -INT <process number>' command. All
of these, will do the job for you: will stop receiving new data from the network,
clear the memory buffers, notify the running plugins to take th exit lane (which
in turn will clear cached data as required).
@@ -301,12 +303,13 @@ A: Few hints are summed below in order to improve SQL database performances. The
in case of unsecured shutdowns (remember power failure is a variable ...).


Q18: I've configured the server hosting pmacct with my local timezone - which includes
DST (Daylight Saving Time). Is this allright?
A: In general, it's good rule to run the backend part of any accounting system as UTC;
pmacct uses the underlying system clock, expecially in the SQL plugins to calculate
time-bins and scanner deadlines among the others. The use of timezones is supported
but not recommended.
Q18: Does having the local timezone configured on servers, routers, etc. - which can
very well include DST (Daylight Saving Time) shifts, impact accounting?
A: It is good rule to run the infrastructure and the backend part of the accounting
system as UTC; for example, accuracy can be negatively impacted if sampled flows
are cached on a router while the DST shift takes place; plus, pmacct uses system
clock to calculate time-bins and scanner deadlines among the others. In short,
the use of local timezones is not recommended.


Q19: I'm using the 'tee' plugin with transparent mode set to true and keep receiving
@@ -323,17 +326,39 @@ A: It means you can't receive packets on an IPv4 address and transparently repli
IP address (nfacctd_ip), if IPv4 is used.


Q20: I've enabled IPv6 support in pmacct with --enable-ipv6. Even though the daemon
binds to the "::" address, i don't receive NetFlow/IPFIX/sFlow/BGP data sent via
IPv4, why?

A: Binding to a "::" address (ie. no [sn]facctd_ip specified when pmacct is compiled
with --enable-ipv6) should allow to receive both IPv4 and IPv6 senders. IPv4 ones
should be reportd in 'netstat' as IPv4-mapped IPv6 addresses. Linux has a kernel
switch to enable/disable the functionality and its status can be checked via the
/proc/sys/net/ipv6/bindv6only . Historically the default has been '0'. It appears
over time some distributions have changed the default to be '1'. If you experience
this issue on Linux, please check your kernel setting.
Q20: I'm using IPv6 support in pmacct. Even though the daemon binds to the "::"
address, i don't receive NetFlow/IPFIX/sFlow/BGP data sent via IPv4, why?

A: Binding to a "::" address (ie. no [sn]facctd_ip specified should allow to receive
both IPv4 and IPv6 senders. IPv4 ones should be reportd in 'netstat' as IPv4-mapped
IPv6 addresses. Linux has a kernel switch to enable/disable the functionality and
its status can be checked via the /proc/sys/net/ipv6/bindv6only . Historically the
default has been '0'. It appears over time some distributions have changed the
default to be '1'. If you experience this issue on Linux, please check your kernel
setting.


Q21: How can i count how much telemetry data (ie. NetFlow, sFlow, IPFIX, Streaming
Telemetry) i'm receiving on my collector?

A: If the interface where telemetry data is received is dedicated to the task then any
ifconfig, netstat or dstat tools or SNMP meaurement would do in order to verify
amount of telemetry packets and bytes (from which packets per second, bytes per
second can be easily inferred). If, instead, the interface is shared then pmacctd,
the libpcap-based daemon, can help to isolate and account for the telemetry traffic;
guess telemetry data is pointed to UDP port 2100 of the IP address configured on
eth0, pmacctd can be started as "pmacctd -i eth0 -P print -c none port 2100" to
account for the grand total of telemetry packets and bytes; if a breakdown per
telemetry exporting node is wanted, the following command-line can be used: "pmacctd
-i eth0 -P print -c src_host port 2100"; this example is suitable for manual reading
as it will print data every 60 secs on the screen and can, of course, be complicated
slightly to make it suitable for automation. A related question that often arises
is: how many flows per second am i receiving? This can be similarly addressed by
using "nfacctd -P print -c flows" for NetFlow/IPFIX and "sfacctd -P print -c flows"
for sFlow. Here FLOWS is the amount of flow records (NetFlow/IPFIX) or flow samples
(sFlow) processed in the period of time, and is the measure of interest. Changing
the aggregation argument in "-c peer_src_ip,flows" gives the amount of flows per
telemetry exporter (ie. router).

/* EOF */

+ 2
- 2
INSTALL View File

@@ -1,5 +1,5 @@
pmacct (Promiscuous mode IP Accounting package)
pmacct is Copyright (C) 2003-2016 by Paolo Lucente
pmacct [IP traffic accounting : BGP : BMP : IGP : Streaming Telemetry]
pmacct is Copyright (C) 2003-2017 by Paolo Lucente

QUICK INSTALLATION:


+ 10
- 3
Makefile.in View File

@@ -57,9 +57,10 @@ DIST_COMMON = $(am__configure_deps) $(srcdir)/Makefile.am \
ltmain.sh missing
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/m4/ac_linearize_path.m4 \
$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/configure.ac
$(top_srcdir)/m4/ax_lib_mysql.m4 $(top_srcdir)/m4/libtool.m4 \
$(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \
$(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \
$(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
@@ -194,6 +195,10 @@ MONGODB_LIBS = @MONGODB_LIBS@
MYSQL_CFLAGS = @MYSQL_CFLAGS@
MYSQL_CONFIG = @MYSQL_CONFIG@
MYSQL_LIBS = @MYSQL_LIBS@
MYSQL_VERSION = @MYSQL_VERSION@
NDPI_CFLAGS = @NDPI_CFLAGS@
NDPI_LIBS = @NDPI_LIBS@
NDPI_LIBS_STATIC = @NDPI_LIBS_STATIC@
NFLOG_CFLAGS = @NFLOG_CFLAGS@
NFLOG_LIBS = @NFLOG_LIBS@
NM = @NM@
@@ -226,6 +231,8 @@ SQLITE3_CFLAGS = @SQLITE3_CFLAGS@
SQLITE3_LIBS = @SQLITE3_LIBS@
STRIP = @STRIP@
VERSION = @VERSION@
ZMQ_CFLAGS = @ZMQ_CFLAGS@
ZMQ_LIBS = @ZMQ_LIBS@
abs_builddir = @abs_builddir@
abs_srcdir = @abs_srcdir@
abs_top_builddir = @abs_top_builddir@

+ 526
- 289
QUICKSTART
File diff suppressed because it is too large
View File


+ 13
- 11
TOOLS View File

@@ -1,11 +1,12 @@
TOOLS DESCRIPTION.

All daemons can print statistics to stdout, keep in memory tables, store
persistently to open-source RDBMS (MySQL, PostgreSQL, Sqlite 3) to noSQL
databates (BerkeleyDB, MongoDB) and to flat-files, and publish to AMQP
and Kafka brokers (typically to insert in ElasticSearch, Cassandra and
CouchDB and, in general, all backends which are not natively supported
by pmacct).
All data plane (ie. IPv4/IPv6 traffic) daemons can print statistics to stdout,
keep them in memory tables, store persistently to open-source RDBMS (MySQL,
PostgreSQL, Sqlite 3) or to noSQL databates (ie. BerkeleyDB) and to flat-files,
and publish to AMQP and Kafka brokers (typically to insert in ElasticSearch,
InfluxDB and Cassandra and, in general, all backends which are not natively
supported by pmacct). BGP, BMP and Streaming Telemetry daemons can publish
control and infrastructure planes to AMQP and Kafka brokers.

pmacctd libpcap-based accounting daemon: it captures packets from an
interface it is bound to. Other than acting as a collector,
@@ -23,13 +24,14 @@ uacctd Linux Netlink NFLOG accounting daemon; it captures packets by
leveraging a NFLOG multicast group - and works only on Linux.
Other than acting as a collector, this daemon can also export
statistics via NetFlow, IPFIX and sFlow protocols.
pmtelemetryd Streaming network telemetry collector daemon; it listens for
pmtelemetryd Standalone Streaming Telemetry collector daemon; listens for
telemetry data binding to a TCP or UDP port and logs real-time
and/or dumps at regular time-intervals to configured backends.
pmbgpd Stand-alone BGP collector daemon; acts as a passive neighbor
and maintains per-peer RIBs; can log real-time and/or dump at
regular time-intervals BGP data to configured backends.
pmbmpd Stand-alone BMP collector daemon; can log real-time and/or dump
pmbgpd Standalone BGP collector daemon; acts as a passive iBGP or
eBGP neighbor and maintains per-peer RIBs; can log real-time
and/or dump at regular time-intervals BGP data to configured
backends.
pmbmpd Standalone BMP collector daemon; can log real-time and/or dump
at regular time-intervals BMP/BGP data to configured backends.
pmacct commandline pmacct client; it allows to retrieve data from a
memory table plugin; it can perform queries over data or do

+ 81
- 7
UPGRADE View File

@@ -1,12 +1,86 @@
UPGRAGE guidelines.
pmacct is developed keeping an eye to backward compatibility: the upgrade to some
newer version should be as smooth as possible from an user standpoint. This is
because, for example, features like SQL table versioning have been introduced
over the time.
pmacct is developed keeping an eye to backward compatibility: the upgrade to
some newer version should be as smooth as possible from an user standpoint.
However, sometimes the upgrade may require some operations aimed to support
the changes done or break old assumptions no longer valid; while the effort
is to keep these cases at a low, please read this file in preparation to
upgrading your installation.

TO: >= 1.7.0
FROM: <= 1.6.2
TOPIC: Obsoleted features
DESC: Following is the list of features, knobs and plugins that are being
discontinued with release 1.7:
* MongoDB plugin is being discontinued since the old Mongo API is
not supported anymore and there has never been enough push from
the community to transition to the new/current API (which would
require a rewrite of most of the plugin)
* Packet classification basing on the L7-filter project is being
discontinued (ie. 'classifiers' directive). This is being replaced
by an implementation basing on the nDPI project. As part of this
also the sql_aggressive_classification knob has been discontinued.
* tee_receiver was part of the original implementation of the tee
plugin, allowing to forward to a single target and hence requiring
multiple plugins instantiated, one per target. Since 0.14.3 this
directive was effectively outdated by tee_receivers.
* tmp_net_own_field knob was allowing to revert to backward compatible
behaviour of IP prefixes (ie. src_net) being written in the same
field as IP addresses (ie. src_host)
* tmp_comms_same_field knob was allowing to revert to backward
compatible behaviour of BGP communities (standard, extended) being
writeen all in the same field.
* plugin_pipe_amqp and plugin_pipe_kafka features were meant as an
alternative to the homegrown queue solution for internal messaging,
ie. passing data from the Core Process to Plugins, and are being
discontinued. They are being replaced by a new implementation,
plugin_pipe_zmq, basing on ZeroMQ.
* plugin_pipe_backlog was allowing to keep an artificial backlog of
data in the Core Process so for plugins to maximise bypass poll()
syscalls in plugins. If home-grown queueing is found limiting,
instead of falling back to such strategies, ZeroMQ queueing should
be used.

TO: >= 1.7.0
FROM: <= 1.6.2
TOPIC: change to sql_num_hosts
DESC: When sql_num_hosts is enabled and pmacct is not compiled with
--disable-ipv6, INET6_ATON() is now used for both IPv4 and IPv6 with
MySQL and SQLite. Tables should be upgraded by changing columns from
INT(4) to VARBINARY(16) and then converting data - for MySQL:

ALTER TABLE table MODIFY COLUMN ip_src VARBINARY(16) NOT NULL;
UPDATE table SET ip_src = INET6_ATON(INET_NTOA(ip_src))
WHERE INET_NTOA(ip_src) IS NOT NULL;

This has performance implications which are mentioned in README.IPv6.
However, sometimes the upgrade may require some easy operations aimed to support
the changes done or break old assumptions no longer valid. Such happenings have
been (and will be) very limited through the development process.
TO: >= 1.6.2
FROM: <= 1.6.1
TOPIC: default plugin names
DESC: Plugin names had to be unique per plugin type, meaning two plugins
could be named "foobar" if they were of different type. Such behaviour
has proven to lead to ambiguous scenarios and hence now plugin names
must be globally unique. If not naming a plugin, its default name
will now be "default_<plugin type>" instead of "default". Any piece
of configuration that attaches directives to the "default" name, ie.
relying on the assumption the plugin name defaults to "default", must
be reviewed.

TO: >= 1.6.2
FROM: <= 1.6.1
TOPIC: print_time_roundoff configuration directive suppressed
DESC: The directive was renamed print_history_roundoff for consistency with
other plugins. print_time_roundoff was already removed from documents
for the past 3+ years.

TO: >= 1.6.2
FROM: <= 1.6.1
TOPIC: sFlow probe (sfprobe plugin) and tags and class primitives
DESC: Historically enterprise #8800 was squatted for the purpose of encoding
pmacct-specific fields in sFlow, ie. tags and class. This never got
changed when pmacct was assigned its own enterprise number (#43874) by
IANA. In 1.6.2, these primitives are moved from #8800 to #43874 making
older exports not compatible anymore.
TO: >= 1.6.1
FROM: <= 1.6.0

+ 1
- 0
aclocal.m4 View File

@@ -1187,6 +1187,7 @@ AC_SUBST([am__untar])
]) # _AM_PROG_TAR

m4_include([m4/ac_linearize_path.m4])
m4_include([m4/ax_lib_mysql.m4])
m4_include([m4/libtool.m4])
m4_include([m4/ltoptions.m4])
m4_include([m4/ltsugar.m4])

+ 4
- 0
bin/configure-help-replace.txt View File

@@ -17,6 +17,8 @@
SQLITE3_LIBS linker flags for SQLITE3, overriding pkg-config
RABBITMQ_CFLAGS C compiler flags for RABBITMQ, overriding pkg-config
RABBITMQ_LIBS linker flags for RABBITMQ, overriding pkg-config
ZMQ_CFLAGS C compiler flags for ZMQ, overriding pkg-config
ZMQ_LIBS linker flags for ZMQ, overriding pkg-config
KAFKA_CFLAGS C compiler flags for KAFKA, overriding pkg-config
KAFKA_LIBS linker flags for KAFKA, overriding pkg-config
GEOIP_CFLAGS C compiler flags for GEOIP, overriding pkg-config
@@ -29,4 +31,6 @@
AVRO_LIBS linker flags for AVRO, overriding pkg-config
NFLOG_CFLAGS C compiler flags for NFLOG, overriding pkg-config
NFLOG_LIBS linker flags for NFLOG, overriding pkg-config
NDPI_CFLAGS C compiler flags for dynamic nDPI, overriding pkg-config
NDPI_LIBS linker flags for dynamic nDPI, overriding pkg-config


+ 742
- 82
configure
File diff suppressed because it is too large
View File


+ 223
- 22
configure.ac View File

@@ -1,7 +1,7 @@
dnl Process this file with autoconf to produce a configure script.
dnl configuration file for pmacct

AC_INIT([pmacct], [1.6.1], [paolo@pmacct.net])
AC_INIT([pmacct], [1.7.0], [paolo@pmacct.net])
AM_INIT_AUTOMAKE([foreign])
AC_CONFIG_MACRO_DIR([m4])
LT_INIT
@@ -9,7 +9,6 @@ AC_PREFIX_DEFAULT([/usr/local])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES(yes)])

COMPILE_ARGS="${ac_configure_args}"
AC_DEFINE_UNQUOTED(COMPILE_ARGS, "$COMPILE_ARGS")

dnl Checks for programs.
AC_PROG_CC
@@ -19,9 +18,7 @@ PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig
export PKG_CONFIG_PATH
PKG_PROG_PKG_CONFIG

host_os=`uname`
host_cpu=`uname -m`
host_os1=`uname -rs`
AC_CANONICAL_HOST

AC_MSG_CHECKING(OS)
AC_MSG_RESULT($host_os)
@@ -117,6 +114,9 @@ case "$host_os" in
*BSD)
AC_DEFINE(BSD, 1)
;;
linux*)
AC_DEFINE(LINUX, 1)
;;
esac

dnl cpu specific flags
@@ -283,12 +283,13 @@ AC_ARG_ENABLE(l2,
[
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_L2, 1)
COMPILE_ARGS="${COMPILE_ARGS} '--enable-l2'"
]
)

AC_MSG_CHECKING([whether to enable IPv6 code])
AC_ARG_ENABLE(ipv6,
[ --enable-ipv6 Enable IPv6 code (default: no)],
[ --enable-ipv6 Enable IPv6 code (default: yes)],
[
if test x$enableval = x"yes" ; then
AC_MSG_RESULT(yes)
@@ -310,8 +311,20 @@ AC_ARG_ENABLE(ipv6,
fi
],
[
AC_MSG_RESULT(no)
ipv6support="no"
AC_MSG_RESULT(yes)
AC_CHECK_FUNCS(inet_pton)
if test x"$ac_cv_func_inet_pton" = x"no"; then
AC_MSG_ERROR(ERROR: missing inet_pton(); disable IPv6 hooks !)
fi

AC_CHECK_FUNCS(inet_ntop)
if test x"$ac_cv_func_inet_ntop" = x"no"; then
AC_MSG_ERROR(ERROR: missing inet_ntop(); disable IPv6 hooks !)
fi

AC_DEFINE(ENABLE_IPV6, 1)
ipv6support="yes"
COMPILE_ARGS="${COMPILE_ARGS} '--enable-ipv6'"
]
)

@@ -331,7 +344,7 @@ AC_ARG_ENABLE(plabel,
])

AC_ARG_WITH(pcap-includes,
[ --with-pcap-includes=DIR Search the specified directories for header files],
[ --with-pcap-includes=DIR Search the specified directory for header files],
[
AC_LINEARIZE_PATH($withval, withval=$absdir)
INCLUDES="${INCLUDES} -I$withval"
@@ -378,7 +391,7 @@ if test x"$PCAPINCLUDESFOUND" = x""; then
fi

AC_ARG_WITH(pcap-libs,
[ --with-pcap-libs=DIR Search the specified directories for libraries],
[ --with-pcap-libs=DIR Search the specified directory for pcap library],
[
AC_LINEARIZE_PATH($withval, withval=$absdir)
LIBS="${LIBS} -L$withval"
@@ -396,6 +409,7 @@ if test x"$PCAPLIB" != x""; then
LIBS="${LIBS} -lpfring -lpcap"
AC_MSG_RESULT(yes)
PFRING_LIB_FOUND=1
AC_DEFINE(PFRING_LIB_FOUND, 1)
else
AC_MSG_RESULT(no)
fi
@@ -416,6 +430,7 @@ if test x"$PCAPLIBFOUND" = x""; then
LIBS="${LIBS} -lpfring -lpcap"
AC_MSG_RESULT(yes)
PFRING_LIB_FOUND=1
AC_DEFINE(PFRING_LIB_FOUND, 1)
else
AC_MSG_RESULT(no)
fi
@@ -432,7 +447,7 @@ if test x"$PFRING_LIB_FOUND" = x""; then
ERROR: missing pcap library. Refer to: http://www.tcpdump.org/
])])

AC_CHECK_LIB([pcap], [pcap_setnonblock], [ AC_DEFINE(PCAP_7, 1) ], [])
AC_CHECK_LIB([pcap], [pcap_set_protocol], [ AC_DEFINE(PCAP_SET_PROTOCOL, 1) ], [])
AC_CHECK_LIB([pcap], [bpf_filter], [ AC_DEFINE(PCAP_NOBPF, 1) ], [])
else
dnl Unable to test: we should check for these libs
@@ -493,7 +508,7 @@ AC_ARG_ENABLE(mysql,
AC_MSG_RESULT(yes)

dnl Unfortunately, no pkg-config support for MySQL
AC_CHECK_PROG([MYSQL_CONFIG], [mysql_config], [mysql_config], [no],,)
AC_CHECK_PROG([MYSQL_CONFIG], [mysql_config], [mysql_config], [no])
if test "x${MYSQL_CONFIG}" = "xno"; then
AC_MSG_ERROR([ERROR: missing mysql_config program])
fi
@@ -502,6 +517,9 @@ dnl Unfortunately, no pkg-config support for MySQL
MYSQL_LIBS=`$MYSQL_CONFIG --libs`],
[AC_MSG_ERROR([ERROR: missing MySQL client library])],
[`$MYSQL_CONFIG --libs`])
dnl version check not enforced with a AC_MSG_ERROR for now
AX_LIB_MYSQL(5.6.3)

AC_SUBST(MYSQL_CFLAGS)
AC_SUBST(MYSQL_LIBS)

@@ -645,6 +663,7 @@ AC_ARG_ENABLE(mongodb,
)
dnl finish: mongodb handling

dnl start: sqlite3 handling
AC_MSG_CHECKING(whether to enable SQLite3 support)
AC_ARG_ENABLE(sqlite3,
[ --enable-sqlite3 Enable SQLite3 support (default: no)],
@@ -677,7 +696,7 @@ AC_ARG_ENABLE(rabbitmq,
yes)
AC_MSG_RESULT(yes)
dnl reasonably old librabbitmq already support pkg-config
PKG_CHECK_MODULES([RABBITMQ], [librabbitmq])
PKG_CHECK_MODULES([RABBITMQ], [librabbitmq >= 0.8.0])
PLUGINS="${PLUGINS} rabbitmq"
USING_RABBITMQ="yes"
PMACCT_CFLAGS="$PMACCT_CFLAGS $RABBITMQ_CFLAGS"
@@ -693,6 +712,28 @@ AC_ARG_ENABLE(rabbitmq,
)
dnl finish: RabbitMQ/AMQP handling

dnl start: ZMQ/AMQP handling
AC_MSG_CHECKING(whether to enable ZMQ/AMQP support)
AC_ARG_ENABLE(zmq,
[ --enable-zmq Enable ZMQ/AMQP support (default: no)],
[ case "$enableval" in
yes)
AC_MSG_RESULT(yes)
PKG_CHECK_MODULES([ZMQ], [libzmq >= 4.2.0])
SUPPORTS="${SUPPORTS} zmq"
USING_ZMQ="yes"
PMACCT_CFLAGS="$PMACCT_CFLAGS $ZMQ_CFLAGS"
AC_DEFINE(WITH_ZMQ, 1)
;;
no)
AC_MSG_RESULT(no)
;;
esac ],
[
AC_MSG_RESULT(no)
]
)
dnl finish: ZMQ/AMQP handling

dnl start: Kafka handling
AC_MSG_CHECKING(whether to enable Kafka support)
@@ -701,7 +742,7 @@ AC_ARG_ENABLE(kafka,
[ case "$enableval" in
yes)
AC_MSG_RESULT(yes)
PKG_CHECK_MODULES([KAFKA], [rdkafka >= 0.8.5],, [
PKG_CHECK_MODULES([KAFKA], [rdkafka >= 0.9.2],, [
AC_MSG_CHECKING([default locations for librdkafka])
if test -r /usr/lib/librdkafka.a -o -r /usr/lib/librdkafka.so; then
KAFKA_LIBS="-L/usr/lib -lrdkafka"
@@ -761,7 +802,7 @@ AC_ARG_ENABLE(geoip,
AC_MSG_RESULT(yes)
dnl reasonably old Maxmind GeoIP v1 already support pkg-config
PKG_CHECK_MODULES([GEOIP], [geoip >= 1.0.0])
PLUGINS="${PLUGINS} geoip"
SUPPORTS="${SUPPORTS} geoip"
USING_MMGEOIP="yes"
PMACCT_CFLAGS="$PMACCT_CFLAGS $GEOIP_CFLAGS"
AC_DEFINE(WITH_GEOIP, 1)
@@ -781,7 +822,7 @@ AC_ARG_ENABLE(geoipv2,
[ case "$enableval" in
yes)
AC_MSG_RESULT(yes)
PKG_CHECK_MODULES([GEOIPV2], [libmaxminddb >= 1.0.0],, [
PKG_CHECK_MODULES([GEOIPV2], [libmaxminddb >= 1.2.0],, [
AC_MSG_CHECKING([default locations for libmaxminddb])
if test -r /usr/lib/libmaxminddb.a -o -r /usr/lib/libmaxminddb.so; then
GEOIPV2_LIBS="-L/usr/lib -lmaxminddb"
@@ -817,7 +858,7 @@ AC_ARG_ENABLE(geoipv2,
CFLAGS="$_save_CFLAGS"
fi
])
PLUGINS="${PLUGINS} geoipv2"
SUPPORTS="${SUPPORTS} geoipv2"
USING_MMGEOIPV2="yes"
PMACCT_CFLAGS="$PMACCT_CFLAGS $GEOIPV2_CFLAGS"
AC_DEFINE(WITH_GEOIPV2, 1)
@@ -840,8 +881,8 @@ AC_ARG_ENABLE(jansson,
yes)
AC_MSG_RESULT(yes)
dnl reasonably old Jansson already support pkg-config
PKG_CHECK_MODULES([JANSSON], [jansson >= 2.2])
PLUGINS="${PLUGINS} jansson"
PKG_CHECK_MODULES([JANSSON], [jansson >= 2.5])
SUPPORTS="${SUPPORTS} jansson"
USING_JANSSON="yes"
PMACCT_CFLAGS="$PMACCT_CFLAGS $JANSSON_CFLAGS"
AC_DEFINE(WITH_JANSSON, 1)
@@ -864,11 +905,12 @@ dnl finish: Jansson handling
dnl start: Avro handling
AC_MSG_CHECKING(whether to enable Avro support)
AC_ARG_ENABLE(avro,
[ --enable-avro Enable avro support (default: no)],
[ --enable-avro Enable Apache Avro support (default: no)],
[ case "$enableval" in
yes)
AC_MSG_RESULT(yes)
PKG_CHECK_MODULES([AVRO], [avro-c >= 1.8])
SUPPORTS="${SUPPORTS} avro"
USING_AVRO="yes"
PMACCT_CFLAGS="$PMACCT_CFLAGS $AVRO_CFLAGS"
AC_DEFINE(WITH_AVRO, 1)
@@ -887,6 +929,74 @@ AC_ARG_ENABLE(avro,
)
dnl finish: Avro handling

dnl start: nDPI handling
AC_ARG_WITH(ndpi-static-lib,
[ --with-ndpi-static-lib=DIR Search the specified directory for nDPI static library],
[
AC_LINEARIZE_PATH($withval, withval=$absdir)
NDPI_CUST_STATIC_LIB=$withval
])

if test x"$NDPI_CUST_STATIC_LIB" != x""; then
AC_MSG_CHECKING(your own nDPI library)
if test -r $NDPI_CUST_STATIC_LIB/libndpi.a; then
AC_MSG_RESULT(ok)
NDPI_CUST_STATIC_LIB_FOUND="yes"
else
AC_MSG_RESULT(no)
AC_MSG_ERROR(ERROR: unable to find nDPI library in $NDPI_CUST_STATIC_LIB)
fi
fi

AC_MSG_CHECKING(whether to enable nDPI support)
AC_ARG_ENABLE(ndpi,
[ --enable-ndpi Enable nDPI support (default: no)],
[ case "$enableval" in
yes)
AC_MSG_RESULT(yes)
PKG_CHECK_MODULES([NDPI], [libndpi >= 2.0])
SUPPORTS="${SUPPORTS} ndpi"
USING_NDPI="yes"

if test x"$NDPI_CFLAGS" != x""; then
NDPI_CFLAGS_INST=`echo $NDPI_CFLAGS | sed 's/ $//'`
NDPI_CFLAGS_INST="$NDPI_CFLAGS_INST/libndpi"
else
NDPI_CFLAGS_INST=""
fi
PMACCT_CFLAGS="$PMACCT_CFLAGS $NDPI_CFLAGS $NDPI_CFLAGS_INST"

AC_DEFINE(WITH_NDPI, 1)
_save_LIBS="$LIBS"
LIBS="$LIBS $NDPI_LIBS"
AC_CHECK_LIB(ndpi, ndpi_init_detection_module, [], [])
LIBS="$_save_LIBS"

dnl XXX: to be improved: avoid linking both static and dynamic libs
if test x"$NDPI_CUST_STATIC_LIB_FOUND" = x"yes"; then
NDPI_LIBS_STATIC="$NDPI_CUST_STATIC_LIB/libndpi.a"
elif test -r /usr/lib/libndpi.a; then
NDPI_LIBS_STATIC="/usr/lib/libndpi.a"
elif test -r /usr/local/lib/libndpi.a; then
NDPI_LIBS_STATIC="/usr/local/lib/libndpi.a"
elif test -r /usr/local/nDPI/lib/libndpi.a; then
NDPI_LIBS_STATIC="/usr/local/nDPI/lib/libndpi.a"
else
AC_MSG_ERROR([ERROR: missing nDPI static library])
fi

AC_SUBST([NDPI_LIBS_STATIC])
;;
no)
AC_MSG_RESULT(no)
;;
esac ],
[
AC_MSG_RESULT(no)
]
)
dnl finish: nDPI handling

if test x"$USING_DLOPEN" = x"yes"; then
AC_DEFINE(HAVE_DLOPEN, 1)
else
@@ -932,6 +1042,7 @@ AC_ARG_ENABLE(64bit,
[
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_64BIT_COUNTERS, 1)
COMPILE_ARGS="${COMPILE_ARGS} '--enable-64bit'"
]
)

@@ -971,6 +1082,7 @@ AC_ARG_ENABLE(threads,

LIBS="${LIBS} -lpthread"
USING_THREADPOOL=yes
COMPILE_ARGS="${COMPILE_ARGS} '--enable-threads'"
]
)

@@ -990,6 +1102,86 @@ AC_ARG_ENABLE(nflog,
esac ],
AC_MSG_RESULT(no))

AC_MSG_CHECKING(whether to link IPv4/IPv6 traffic accounting accounting binaries)
AC_ARG_ENABLE(traffic-bins,
[ --enable-traffic-bins Link IPv4/IPv6 traffic accounting binaries (default: yes)],
[
if test x$enableval = x"yes" ; then
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_TRAFFIC_BINS, 1)
USING_TRAFFIC_BINS="yes"
else
AC_MSG_RESULT(no)
fi
],
[
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_TRAFFIC_BINS, 1)
USING_TRAFFIC_BINS="yes"
COMPILE_ARGS="${COMPILE_ARGS} '--enable-traffic-bins'"
]
)

AC_MSG_CHECKING(whether to link BGP daemon binaries)
AC_ARG_ENABLE(bgp-bins,
[ --enable-bgp-bins Link BGP daemon binaries (default: yes)],
[
if test x$enableval = x"yes" ; then
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BGP_BINS, 1)
USING_BGP_BINS="yes"
else
AC_MSG_RESULT(no)
fi
],
[
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BGP_BINS, 1)
USING_BGP_BINS="yes"
COMPILE_ARGS="${COMPILE_ARGS} '--enable-bgp-bins'"
]
)

AC_MSG_CHECKING(whether to link BMP daemon binaries)
AC_ARG_ENABLE(bmp-bins,
[ --enable-bmp-bins Link BMP daemon binaries (default: yes)],
[
if test x$enableval = x"yes" ; then
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BMP_BINS, 1)
USING_BMP_BINS="yes"
else
AC_MSG_RESULT(no)
fi
],
[
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_BMP_BINS, 1)
USING_BMP_BINS="yes"
COMPILE_ARGS="${COMPILE_ARGS} '--enable-bmp-bins'"
]
)

AC_MSG_CHECKING(whether to link Streaming Telemetry daemon binaries)
AC_ARG_ENABLE(st-bins,
[ --enable-st-bins Link Streaming Telemetry daemon binaries (default: yes)],
[
if test x$enableval = x"yes" ; then
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_ST_BINS, 1)
USING_ST_BINS="yes"
else
AC_MSG_RESULT(no)
fi
],
[
AC_MSG_RESULT(yes)
AC_DEFINE(HAVE_ST_BINS, 1)
USING_ST_BINS="yes"
COMPILE_ARGS="${COMPILE_ARGS} '--enable-st-bins'"
]
)

dnl Checks for library functions.
AC_TYPE_SIGNAL

@@ -997,6 +1189,7 @@ AC_CHECK_FUNCS([strlcpy vsnprintf setproctitle mallopt tdestroy])

dnl final checks
dnl trivial solution to portability issue
AC_DEFINE_UNQUOTED(COMPILE_ARGS, "$COMPILE_ARGS")
CFLAGS="${CFLAGS} ${INCLUDES}"
INCLUDES=""

@@ -1008,6 +1201,7 @@ CFLAGS ....... : ${CFLAGS}
LIBS ......... : ${LIBS}
LDFLAGS ...... : ${LDFLAGS}
PLUGINS ...... : ${PLUGINS}
SUPPORTS ..... : ${SUPPORTS}

Now type 'make' to compile the source code.

@@ -1030,13 +1224,20 @@ AM_CONDITIONAL([WITH_PGSQL], [test x"$USING_PGSQL" = x"yes"])
AM_CONDITIONAL([WITH_MONGODB], [test x"$USING_MONGODB" = x"yes"])
AM_CONDITIONAL([WITH_SQLITE3], [test x"$USING_SQLITE3" = x"yes"])
AM_CONDITIONAL([WITH_RABBITMQ], [test x"$USING_RABBITMQ" = x"yes"])
AM_CONDITIONAL([WITH_ZMQ], [test x"$USING_ZMQ" = x"yes"])
AM_CONDITIONAL([WITH_KAFKA], [test x"$USING_KAFKA" = x"yes"])
AM_CONDITIONAL([USING_SQL], [test x"$USING_SQL" = x"yes"])
AM_CONDITIONAL([USING_THREADPOOL], [test x"$USING_THREADPOOL" = x"yes"])
AM_CONDITIONAL([WITH_NFLOG], [test x"$USING_NFLOG" = x"yes"])
AM_CONDITIONAL([WITH_AVRO], [test x"$USING_AVRO" = x"yes"])
AM_CONDITIONAL([WITH_NDPI], [test x"$USING_NDPI" = x"yes"])
AM_CONDITIONAL([WITH_NFLOG], [test x"$USING_NFLOG" = x"yes"])
AM_CONDITIONAL([USING_TRAFFIC_BINS], [test x"$USING_TRAFFIC_BINS" = x"yes"])
AM_CONDITIONAL([USING_BGP_BINS], [test x"$USING_BGP_BINS" = x"yes"])
AM_CONDITIONAL([USING_BMP_BINS], [test x"$USING_BMP_BINS" = x"yes"])
AM_CONDITIONAL([USING_ST_BINS], [test x"$USING_ST_BINS" = x"yes"])
AC_OUTPUT([ Makefile \
src/Makefile src/nfprobe_plugin/Makefile \
src/sfprobe_plugin/Makefile src/bgp/Makefile \
src/tee_plugin/Makefile src/isis/Makefile \
src/bmp/Makefile src/telemetry/Makefile ])
src/bmp/Makefile src/telemetry/Makefile \
src/ndpi/Makefile ])

+ 19
- 18
docs/INTERNALS View File

@@ -1,4 +1,4 @@
(poorman's) TABLE OF CONTENTS:
TABLE OF CONTENTS:
I. Introduction
II. Primitives
III. The whole picture
@@ -13,13 +13,10 @@ XI. BGP daemon thread dimensioning


I. Introduction
Giving a quick look to the old 'INTERNALS' textfile, this new one starts with a big step
forward: a rough table of contents, though the document is still not fancy nor formatted.
I'm also conscious the package is still missing its man page. The goal of this document
would be an 'as much as possible' careful description of the development paths, trying to
expose the work done to constructive critics.
Since March 2005, this document is complemented by a paper about an architectural overview
of the project 'pmacct: steps forward interface counters'; the referred paper is available
The goal of this document would be to give extra insight on some of the internals of
pmacct (useful for development or simply constructive critics). Since March 2005,
this document is complemented by a paper about an architectural overview of the
project 'pmacct: steps forward interface counters'; the referred paper is available
for download at the pmacct homepage.


@@ -74,7 +71,7 @@ NetFlow | \ && | |
| |
| [ handle ] [ handle ] [ handle ] [ handle ] [ handle ] |
| ... ====[ link layer ]=====[ IP layer ]====[ fragments ]==== [ flows ]==== [ classification ] ... |
| ll.c nl.c ip_frag.c ip_flow.c classifier.c |
| ll.c nl.c ip_frag.c ip_flow.c nDPI library |
| |
| [ handle ] [ Correlate ] |
| ... ====[ maps ]===== [ BGP, IGP ] ... |
@@ -217,6 +214,7 @@ when dealing with a 'dynamic' memory table (which is allowed to grow undefinitel
memory new chunks of memory are allocated and added to the list during the execution.
Using a fixed table places a maximum limit to the number of entries the table is able
to store; the following calculation may help in building a fixed table:

ES (Entry Size) ~ 50 bytes
NE (Number of entries)

@@ -224,6 +222,10 @@ NE (Number of entries)

Default values are: imt_mem_pools_number = 16; imt_mem_pools_size = 8192; this will let
the default fixed table to contain a maximum of slightly more than 2600 aggregates.
However note the entry size is indicative and can very consistently, ie. depending if
IPv6 or Layer2 are enabled at compile time or whether BGP, MPLS, NAT, etc. primitives
are in use as part of the aggregation key. When a fixed size table is needed, it is
better to constrain it on the size rather than the estimated number of entries to fit.
IMT plugin does not rely any way over the realloc() function, but only mmap(). Table
grows and shrinks with the help of the above described tracking structures. This is
@@ -356,9 +358,10 @@ accounting is enabled (ie. print plugin, 'sql_history', etc.).


IX. Classifier and connection tracking engines
pmacct 0.10.0 sees the introduction of new packet/stream classification and connection tracking
features in the pmacctd daemon. Firstly, let's give a look to the global picture; then how they
work:
Classification and connection tracking features were introduced in pmacctd and uacctd daemons as
early as 0.10.0 release. As of pmacct 1.7, classification is switched from the mixed home-grown
implementation + L7 layer project to the nDPI library. Firstly, let's give a look to the global
picture; then how they work:

----[ pmacctd loop ]-------------------------------------------------------------
| [ regular ] |
@@ -371,7 +374,7 @@ work:
| [ fragment ] [ flow ] [ flow ] [ connection ] |
| ... ==>[ handling ]==>[ handling ]==>[ classification ]==>[ tracking ]==> ... |
| [ engine ] [ engine ] [ engine ] [ engine ] |
| ip_frag.c ip_flow.c classifier.c \ conntrack.c |
| ip_frag.c ip_flow.c nDPI library \ conntrack.c |
| | \___ |
| \ \ |
| \ [ shared ] |
@@ -393,11 +396,9 @@ A connection tracking module might be assigned to certain classified streams if
a protocol which is known to be based over a control channel (ie. FTP, RTSP, SIP, H.323, etc.).
However, some protocols (ie. MSN messenger) spawn data channels that can still be distinguished
because of some regular patterns into the payload; in such cases a classificator exists rather
than a tracking module. Connection tracking modules are C routines statically compiled into the
collector code that hint IP address/port couples for upcoming data streams as signalled by one
of the parties into the control channel; such information fragments are then meant to classify
the new data streams; classification patterns are either regular expressions (RE) or pluggable
shared objects (SO, written in C), both loaded at runtime.
than a tracking module. Connection tracking modules hint IP address/port couples for upcoming
data streams as signalled by one of the parties into the control channel; such pieces of
information are then meant to classify the new data streams.
In this context, 'snaplen' directive, which specifies the maximum number of bytes to capture for
each packet, has key importance. In fact, some protocols (mostly text-based eg. RTSP, SIP, etc.)
benefit of extra bytes because they give more chances to identify new data streams spawned by

+ 200
- 83
docs/MSGLOG_DUMP_FORMATS View File

@@ -5,100 +5,217 @@ UPGRADE document to verify if any impacting changes to the message formats were
introduced.

BGP msglog format:
* log_init message:
{"seq": <seq>, "timestamp": <timestamp>, "peer_ip_src": <IP address>, \
"event_type": "log_init"}

* log message:
{"seq": <seq>, "timestamp": <timestamp>, "peer_ip_src": <IP address>, \
"event_type": "log", "log_type": <"update", "withdraw", "delete">, \
<BGP NLRI, attributes, ..>}

* log_close message:
{"seq": <seq>, "timestamp": <timestamp>, "peer_ip_src": <IP address>, \
"event_type": "log_close"}
- log_init message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"peer_ip_src": <IP address>,
"event_type": "log_init"
}

- log message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"peer_ip_src": <IP address>,
"event_type": "log",
"afi": <afi>,
"safi": <safi>,
"log_type": <"update", "withdraw", "delete">,
<BGP NLRI, attributes, ..>
}

- log_close message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"peer_ip_src": <IP address>,
"event_type": "log_close"
}


BGP dump format:
* dump_init message:
{"timestamp": <dump event timestamp>, "peer_ip_src": <IP address>, \
"event_type": "dump_init", "dump_period": <bgp_table_dump_refresh_time>}

* dump message:
{"timestamp": <dump event timestamp>, "peer_ip_src": <IP address>, \
"event_type": "dump", <BGP NLRI, attributes, ..>}

* dump_close message:
{"timestamp": <dump event timestamp>, "peer_ip_src": <IP address>, \
"event_type": "dump_close"}
- dump_init message:
{
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"peer_ip_src": <IP address>,
"event_type": "dump_init",
"dump_period": <bgp_table_dump_refresh_time>
}

- dump message:
{
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"peer_ip_src": <IP address>,
"event_type": "dump",
"afi": <afi>,
"safi": <safi>,
<BGP NLRI, attributes, ..>
}

- dump_close message:
{
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"peer_ip_src": <IP address>,
"event_type": "dump_close"
}


BMP msglog format:
* log_init message:
{"seq": <seq>, "timestamp": <timestamp>, "bmp_router": <IP address>, \
"event_type": "log_init"}

* log routes message:
{"seq": <seq>, "timestamp": <BMP timestamp>, "bmp_router": <IP address>, \
"event_type": "log", "bmp_msg_type": "route_monitor", "log_type": <"update", \
"withdraw", "delete">, "peer_ip": <IP address>, <BGP NLRI, attributes, ..>}

* log events message:
{"seq": <seq>, "timestamp": <BMP timestamp>, "bmp_router": <IP address>, \
"event_type": "log", "bmp_msg_type": <"init", "term", "peer_up", "stats", \
"peer_down">, "peer_ip": <IP address>, <BMP message data>}

* log_close message:
{"seq": <seq>, "timestamp": <timestamp>, "bmp_router": <IP address>, \
"event_type": "log_close"}
- log_init message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"bmp_router": <IP address>,
"event_type": "log_init"
}

- log routes message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <BMP timestamp>,
"bmp_router": <IP address>,
"event_type": "log",
"afi": <afi>,
"safi": <safi>,
"bmp_msg_type": "route_monitor", "log_type": <"update", "withdraw", "delete">,
"peer_ip": <IP address>,
<BGP NLRI, attributes, ..>
}

- log events message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <BMP timestamp>,
"bmp_router": <IP address>,
"event_type": "log",
"bmp_msg_type": <"init", "term", "peer_up", "stats", "peer_down">,
"peer_ip": <IP address>,
<BMP message data>
}

- log_close message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"bmp_router": <IP address>,
"event_type": "log_close"
}


BMP dump format:
* dump_init message:
{"timestamp": <dump event timestamp>, "bmp_router": <IP address>, \
"event_type": "dump_init", "dump_period": <bmp_dump_refresh_time>}

* dump routes message:
{"timestamp": <dump event timestamp>, "bmp_router": <IP address>, \
"bmp_msg_type": "route_monitor", "event_type": "dump", \
"peer_ip": <IP address>, <BGP NLRI, attributes, ..>}

* dump events message:
{"seq": <seq>, "timestamp": <dump event timestamp>, "bmp_router": <IP address>, \
"event_type": "dump", "event_timestamp": <BMP timestamp>, "bmp_msg_type": \
<"init", "term", "peer_up", "stats", "peer_down">, "peer_ip": <IP address>, \
<BMP message data>}

* dump_close message:
{"timestamp": <dump event timestamp>, "bmp_router": <IP address>, \
"event_type": "dump_close"}
- dump_init message:
{
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"bmp_router": <IP address>,
"event_type": "dump_init",
"dump_period": <bmp_dump_refresh_time>
}

- dump routes message:
{
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"bmp_router": <IP address>,
"bmp_msg_type": "route_monitor",
"event_type": "dump",
"afi": <afi>,
"safi": <safi>,
"peer_ip": <IP address>,
<BGP NLRI, attributes, ..>
}

- dump events message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"bmp_router": <IP address>,
"event_type": "dump",
"event_timestamp": <BMP timestamp>,
"bmp_msg_type": <"init", "term", "peer_up", "stats", "peer_down">,
"peer_ip": <IP address>,
<BMP message data>
}

- dump_close message:
{
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"bmp_router": <IP address>,
"event_type": "dump_close"
}


Streaming Telemetry msglog format:
* log_init message:
{"seq": <seq>, "timestamp": <timestamp>, "telemetry_node": <IP address>, \
"event_type": "log_init"}

* log message:
{"seq": <seq>, "timestamp": <timestamp>, "telemetry_node": <IP address>, \
"event_type": "log", "telemetry_port": <TCP/UDP port>, "serialization": \
<"json" | "gpb">, "telemetry_data": <JSON or base64'd GPB telemetry data>}

* log_close message:
{"seq": <seq>, "timestamp": <timestamp>, "telemetry_node": <IP address>, \
"event_type": "log_close"}
- log_init message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"telemetry_node": <IP address>,
"event_type": "log_init"
}

- log message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"telemetry_node": <IP address>,
"event_type": "log",
"telemetry_port": <TCP/UDP port>,
"serialization": <"json" | "gpb">,
"telemetry_data": <JSON or base64'd GPB telemetry data>
}

- log_close message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <timestamp>,
"telemetry_node": <IP address>,
"event_type": "log_close"
}

Streaming Telemetry dump format:
* dump_init message:
{"timestamp": <dump event timestamp>, "telemetry_node": <IP address>, \
"event_type": "dump_init", "dump_period": <bmp_dump_refresh_time>}

* dump message:
{"seq": <seq>, "timestamp": <dump event timestamp>, "telemetry_node": \
<IP address>, "event_type": "dump", "telemetry_port": <TCP/UDP port>, \
"serialization": <"json" | "gpb">, "telemetry_data": <JSON or base64'd \
GPB telemetry data>}

* dump_close message:
{"timestamp": <dump event timestamp>, "telemetry_node": <IP address>, \
"event_type": "dump_close"}
- dump_init message:
{
"timestamp": <dump event timestamp>,
"telemetry_node": <IP address>,
"event_type": "dump_init",
"dump_period": <bmp_dump_refresh_time>
}

- dump message:
{
"seq": <seq>,
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"telemetry_node": <IP address>,
"event_type": "dump",
"telemetry_port": <TCP/UDP port>,
"serialization": <"json" | "gpb">,
"telemetry_data": <JSON or base64'd GPB telemetry data>
}

- dump_close message:
{
"writer_id": <core_proc_name>"/"<writer_pid> (*),
"timestamp": <dump event timestamp>,
"telemetry_node": <IP address>,
"event_type": "dump_close"
}

(*) Field included only when writing to a RabbitMQ or Kafka broker

+ 12
- 9
docs/SIGNALS View File

@@ -1,10 +1,13 @@
SIGNALS:
Here follows a list of supported signals and their meaning; note: pmacct core
process says goodbye when its last child dies or is terminated.
Here follows a list of supported signals and their meaning. If a signal is
not being properly delivered to the daemon, and this is on a system running
SELinux, check for SELinux interferences.


Core process:
SIGCHLD: used to handle gracefully his loved child processes;
SIGCHLD: used to handle gracefully his loved child processes. This is
internal, ie. should not be sent by users. To end gracefully
the daemon, look at SIGINT;
SIGHUP: reopens the logging infrastructure. Works with both syslog and
logfiles; it also works with streamed logging of BGP messages/
events (bgp_daemon_msglog_file), streamed logging of BMP data/
@@ -17,12 +20,12 @@ SIGUSR1: returns various statistics via either console or syslog; the
SIGUSR2: if 'maps_refresh' config directive is enabled, it causes maps
to be reloaded (ie. pre_tag_map, bgp_agent_map, etc.). If also
indexing is enabled, ie. maps_index, indexes are re-compited.
SIGINT: if starting pmacct in foreground the signal is propagated to
each running plugin, which is in turn gracefully terminated;
if starting pmacct in background, this signal is ignored by
the Core process but not from the plugins: it is recommended
to send the signal to all plugins, ie. "killall -INT pmacctd"
so to let the whole pmacct instance exit gracefully.
SIGINT: the signal is used by the Core Process itself and propagated
to each running plugin for graceful termination (ie. send BGP
NOTIFICATION message to established BGP sessions, close open
files, remove PID files, purge data, etc.). See Q16 of the
FAQS document for recommendations on how to best send SIGINT
signals to the daemon;
SIGTERM: not handled (which means it follows the default behaviour for
the OS) if the daemon is started in background; otherwise it
orks like SIGINT;

+ 9
- 10
docs/TRIGGER_VARS View File

@@ -1,14 +1,13 @@
INTRODUCTION
An executable triggering mechanism feature is part of all SQL plugins
(sql_trigger_exec). Executables may either be spawned each time a cache
purging event occurs or at arbitrary time intervals (that are specified
via sql_trigger_time). Because the triggering mechanism is hooked on top
of the 'lazy deadlines' plugin concept, it should not be preferred method
to run tasks strictly connected to timing issues (use crontab instead).
As a recap, the concept of lazy deadlines was introduced a while ago to
avoid large use of UNIX signals for precise time handling. Information
is being passed to the triggered executable in the form of environment
variables. The list of supported variables follows:
A feature to spawn external executables is part of all pmacct plugins
(ie. sql_trigger_exec, print_trigger_exec, etc). In case of SQL plugins,
executables may either be spawned each time a cache purging event occurs
or at arbitrary time intervals (specified via sql_trigger_time); in all
other plugins a trigger can be spawned only at a cache purging event.
For time-sensitive triggers it is recommended to use crontab instead.
Also, in case of SQL plugins some information is being passed to the
triggered executable in the form of environment variables. The list of
supported variables follows:

VAR: $SQL_DB
DESC: RDBMS database name.

+ 15
- 6
examples/agent_to_peer.map.example View File

@@ -8,7 +8,7 @@
!
! list of currently supported keys follow:
!
! 'bgp_ip' LOOKUP: IPv4/IPv6 session address or router ID of the
! 'bgp_ip' LOOKUP: IPv4/IPv6 session address or Router ID of the
! BGP peer.
! 'bgp_port' LOOKUP: TCP port used by the BGP peer to establish the
! session, useful in NAT traversal scenarios.
@@ -24,12 +24,12 @@
! 'filter' MATCH: incoming data is compared against the supplied
! filter expression (expected in libpcap syntax); the
! filter needs to be enclosed in quotes ('). In this map
! this is meant to discriminate among IPv4 ('ip') and
! IPv6 ('ip6') traffic.
! this is meant to discriminate among IPv4 ('ip', 'vlan
! and ip') and IPv6 ('ip6', 'vlan and ip6') traffic.
!
! A couple of straightforward examples follow.
!
bgp_ip=1.2.3.4 ip=2.3.4.5
bgp_ip=1.2.3.4 ip=2.3.4.5
!
! The following maps something which any Netflow/sFlow agent to the specified
! BGP peer. This syntax applies also to non-telemetry daemons, ie. pmacctd and
@@ -38,7 +38,16 @@ bgp_ip=1.2.3.4 ip=2.3.4.5
! bgp_ip=4.5.6.7 ip=0.0.0.0/0
!
! The following maps flows ingressing a specific interface of the NetFlow/sFlow
! agent to the specified BGP peer. This is relevant to VPN scenarios.
! agent to the specified BGP peer. This may be relevant to MPLS VPN scenarios.
!
bgp_ip=1.2.3.4 ip=2.3.4.5 in=100
! bgp_ip=1.2.3.4 ip=2.3.4.5 in=100
!
! In scenarios where there are distinct v4 and v6 BGP sessions with the same
! peer (by design or due to distinct BGP agents for v4 and v6), traffic can
! be directed onto the right session with a filter. pmacct needs somehow to
! distinguish the sessions to make the correlation properly work: if the IP
! address of the BGP sessions is the same, ie. pmacct is co-located with the
! BGP agent, the peers will need to have a different Router ID configured:
!
! bgp_ip=4.0.0.1 ip=0.0.0.0/0 filter='ip or (vlan and ip)'
! bgp_ip=6.0.0.1 ip=0.0.0.0/0 filter='ip6 or (vlan and ip6)'

+ 139
- 15
examples/amqp/amqp_receiver.py View File

@@ -1,17 +1,20 @@
#!/usr/bin/env python
#
# If missing 'pika' read how to download it at:
# Pika is a pure-Python implementation of the AMQP 0-9-1 protocol and
# is available at:
# https://pypi.python.org/pypi/pika
# http://www.rabbitmq.com/tutorials/tutorial-one-python.html
#
# UltraJSON, an ultra fast JSON encoder and decoder, is available at:
# https://pypi.python.org/pypi/ujson
#
# The Apache Avro Python module is available at:
# https://avro.apache.org/docs/1.8.1/gettingstartedpython.html
#
# Binding to the routing key specified by amqp_routing_key (by default 'acct')
# allows to receive messages published by an 'amqp' plugin, in JSON format.
# Similarly for BGP daemon bgp_*_routing_key and BMP daemon bmp_*_routing_key.
#
# Binding to the routing key specified by plugin_pipe_amqp_routing_key (by
# default 'core_proc_name-$plugin_name-$plugin_type') allows to receive a copy
# of messages published by the Core Process to a specific plugin; the messages
# are in binary format, first quad being the sequence number.
#
# Binding to the reserved exchange 'amq.rabbitmq.trace' and to routing keys
# 'publish.pmacct' or 'deliver.<queue name>' allows to receive a copy of the
# messages that published via a specific exchange or delivered to a specific
@@ -20,8 +23,17 @@
#
# 'rabbitmqctl trace_on' enables RabbitMQ Firehose tracer
# 'rabbitmqctl list_queues' lists declared queues
#
# Two pipelines are supported in this script:
# * RabbitMQ -> REST API
# * RabbitMQ -> stdout
#
# Two data encoding formats are supported in this script:
# * JSON
# * Apache Avro

import sys, os, getopt, pika, StringIO
import sys, os, getopt, pika, StringIO, time
import ujson as json

try:
import avro.io
@@ -32,6 +44,14 @@ except ImportError:
avro_available = False

avro_schema = None
http_url_post = None
print_stdout = 0
print_stdout_num = 0
print_stdout_max = 0
convert_to_json_array = 0
stats_interval = 0
time_count = 0
elem_count = 0

def usage(tool):
print ""
@@ -46,27 +66,114 @@ def usage(tool):
print "Optional Args:"
print " -h, --help".ljust(25) + "Print this help"
print " -H, --host".ljust(25) + "Define RabbitMQ broker host [default: 'localhost']"
print " -p, --print".ljust(25) + "Print data to stdout"
print " -n, --num".ljust(25) + "Number of rows to print to stdout [default: 0, ie. forever]"
print " -u, --url".ljust(25) + "Define a URL to HTTP POST data to"
print " -a, --to-json-array".ljust(25) + "Convert list of newline-separated JSON objects in a JSON array"
print " -s, --stats-interval".ljust(25) + "Define a time interval, in secs, to get statistics to stdout"
if avro_available:
print " -d, --decode-with-avro".ljust(25) + "Define the file with the " \
"schema to use for decoding Avro messages"

def post_to_url(http_req, value):
try:
urllib2.urlopen(http_req, value)
except urllib2.HTTPError, err:
print "WARN: urlopen() returned HTTP error code:", err.code
sys.stdout.flush()
except urllib2.URLError, err:
print "WARN: urlopen() returned URL error reason:", err.reason
sys.stdout.flush()

def callback(ch, method, properties, body):
global avro_schema
global http_url_post
global print_stdout
global print_stdout_num
global print_stdout_max
global convert_to_json_array
global stats_interval