* Introducing nfacctd_account_options: if set to true account for Net…

…Flow/

  IPFIX option records as well as flow ones. pre_tag_map offers sample_type
  value of 'option' now to split option data records from flow ones.

CONFIG-KEYS

@@ -923,7 +923,7 @@ DESC:		makes 'nfacctd' to ignore timestamps included in NetFlow header and build
 		('sql_history') and an INSERT-only mechanism is in use ('sql_dont_try_update', 'sql_use_copy').
 		(default: false) 
 
-KEY:		nfacctd_pro_rating [ NO_PMACCTD]
+KEY:		nfacctd_pro_rating [NO_PMACCTD]
 VALUES:		[ true | false ]
 DESC:		if nfacctd_time_new is set to false (default) and historical accounting (ie. sql_history) is
 		enabled, this directive enables pro rating of NetFlow/IPFIX flows over time-bins, if needed.
@@ -935,6 +935,12 @@ DESC:		if nfacctd_time_new is set to false (default) and historical accounting (
 NOTES:          If NetFlow sampling is enabled, it is recommended to have counters renormalization enabled
                 (nfacctd_renormalize set to true).
 
+KEY:            nfacctd_account_options [GLOBAL, NO_PMACCTD]
+VALUES:         [ true | false ]
+DESC:		if set to true account for NetFlow/IPFIX option records. This will require define custom
+		primitives via aggregate_primitives. pre_tag_map offers sample_type value of 'option' in
+		order to split option data records from flow or event data ones. (default: false)  
+
 KEY:		[ nfacctd_as_new | sfacctd_as_new | pmacctd_as | uacctd_as ] [GLOBAL]
 VALUES:		[ false | (true|file) | bgp | fallback ]
 DESC:		When 'false', it instructs nfacctd and sfacctd to populate 'src_as', 'dst_as', 'peer_src_as' and

examples/pretag.map.example

@@ -83,10 +83,11 @@
 !			NetFlow v9 and IPFIX are unsupported instead.
 ! 'sample_type'         MATCH: in sFlow v2/v4/v5 this is compared against the
 !			sample type field. Expected in <Enterprise>:<Format>
-!			notation. In NetFlow/IPIX two keywords are supported:
+!			notation. In NetFlow/IPIX three keywords are supported:
 !			"flow" to denote templates suitable to transport flow
-!			traffic data and "event" to denote templates suitable to
-!			flag events. 
+!			traffic data, "event" to denote templates suitable to
+!			flag events and "option" to denote NetFlow/IPFIX option
+!			records data.
 ! 'direction'		MATCH: In NetFlow v9 and IPFIX this is compared against
 !			the direction (61) field, which only valid values are 0
 !			(ingress) and 1 (egress) flow.

src/cfg.h

@@ -150,6 +150,7 @@ struct configuration {
   char *nfacctd_allow_file;
   int nfacctd_time;
   int nfacctd_pro_rating;
+  int nfacctd_account_options;
   u_int32_t nfacctd_as;
   u_int32_t nfacctd_net;
   u_int64_t nfacctd_pipe_size;

src/cfg_handlers.c

@@ -1624,6 +1624,20 @@ int cfg_key_nfacctd_pro_rating(char *filename, char *name, char *value_ptr)
   return changes;
 }
 
+int cfg_key_nfacctd_account_options(char *filename, char *name, char *value_ptr)
+{
+  struct plugins_list_entry *list = plugins_list;
+  int value, changes = 0;
+
+  value = parse_truefalse(value_ptr);
+  if (value < 0) return ERR;
+
+  for (; list; list = list->next, changes++) list->cfg.nfacctd_account_options = value;
+  if (name) Log(LOG_WARNING, "WARN ( %s ): plugin name not supported for key 'nfacctd_account_options'. Globalized.\n", filename);
+
+  return changes;
+}
+
 int cfg_key_nfacctd_bgp_pipe_size(char *filename, char *name, char *value_ptr)
 {
   struct plugins_list_entry *list = plugins_list;

src/cfg_handlers.h

@@ -113,6 +113,7 @@ EXT int cfg_key_nfacctd_disable_checks(char *, char *, char *);
 EXT int cfg_key_nfacctd_mcast_groups(char *, char *, char *);
 EXT int cfg_key_nfacctd_pipe_size(char *, char *, char *);
 EXT int cfg_key_nfacctd_pro_rating(char *, char *, char *);
+EXT int cfg_key_nfacctd_account_options(char *, char *, char *);
 EXT int cfg_key_pmacctd_force_frag_handling(char *, char *, char *);
 EXT int cfg_key_pmacctd_frag_buffer_size(char *, char *, char *);
 EXT int cfg_key_pmacctd_flow_buffer_size(char *, char *, char *);

src/mongodb_plugin.c

@@ -612,7 +612,7 @@ void MongoDB_cache_purge(struct chained_cache *queue[], int index)
         bson_append_date(bson_elem, "stamp_updated", (bson_date_t) 1000*time(NULL));
       }
 
-      if (queue[j]->flow_type != NF9_FTYPE_EVENT) {
+      if (queue[j]->flow_type != NF9_FTYPE_EVENT && queue[j]->flow_type != NF9_FTYPE_OPTION) {
   #if defined HAVE_64BIT_COUNTERS
         bson_append_long(bson_elem, "packets", queue[j]->packet_counter);
         if (config.what_to_count & COUNT_FLOWS) bson_append_long(bson_elem, "flows", queue[j]->flow_counter);

src/mysql_plugin.c

@@ -253,7 +253,7 @@ int MY_cache_dbop(struct DBdesc *db, struct db_cache *cache_elem, struct insert_
   for (num = 0; num < idata->num_primitives; num++)
     (*where[num].handler)(cache_elem, idata, num, &ptr_values, &ptr_where);
 
-  if (cache_elem->flow_type == NF9_FTYPE_EVENT) {
+  if (cache_elem->flow_type == NF9_FTYPE_EVENT || cache_elem->flow_type == NF9_FTYPE_OPTION) {
     for (num_set = 0; set_event[num_set].type; num_set++)
       (*set_event[num_set].handler)(cache_elem, idata, num_set, &ptr_set, NULL);
   }
@@ -275,7 +275,7 @@ int MY_cache_dbop(struct DBdesc *db, struct db_cache *cache_elem, struct insert_
 
   if (config.sql_dont_try_update || !num_set || (mysql_affected_rows(db->desc) == 0)) {
     /* UPDATE failed, trying with an INSERT query */ 
-    if (cache_elem->flow_type == NF9_FTYPE_EVENT) {
+    if (cache_elem->flow_type == NF9_FTYPE_EVENT || cache_elem->flow_type == NF9_FTYPE_OPTION) {
       strncpy(insert_full_clause, insert_clause, SPACELEFT(insert_full_clause));
       strncat(insert_full_clause, insert_nocounters_clause, SPACELEFT(insert_full_clause));
       strncat(ptr_values, ")", SPACELEFT(values_clause));

src/nfacctd.c

@@ -1374,6 +1374,15 @@ void process_v9_packet(unsigned char *pkt, u_int16_t len, struct packet_ptrs_vec
           }
 	}
 
+	if (config.nfacctd_account_options) {
+	  pptrs->f_data = pkt;
+	  pptrs->f_tpl = (u_char *) tpl;
+	  reset_net_status_v(pptrsv);
+	  pptrs->flow_type = NF_evaluate_flow_type(tpl, pptrs);
+
+	  exec_plugins(pptrs, req);
+	}
+
         pkt += tpl->len;
         flowoff += tpl->len;
 
@@ -2066,6 +2075,9 @@ u_int16_t NF_evaluate_flow_type(struct template_cache_entry *tpl, struct packet_
   /* NetFlow Event Logging (NEL): generic NAT event support */
   if (tpl->tpl[NF9_NAT_EVENT].len) ret = NF9_FTYPE_NAT_EVENT;
 
+  /* NetFlow/IPFIX option */
+  if (tpl->template_type == 1) ret = NF9_FTYPE_OPTION;
+
   return ret;
 }
 

src/pgsql_plugin.c

@@ -299,7 +299,7 @@ int PG_cache_dbop(struct DBdesc *db, struct db_cache *cache_elem, struct insert_
   for (num = 0; num < idata->num_primitives; num++)
     (*where[num].handler)(cache_elem, idata, num, &ptr_values, &ptr_where);
 
-  if (cache_elem->flow_type == NF9_FTYPE_EVENT) {
+  if (cache_elem->flow_type == NF9_FTYPE_EVENT || cache_elem->flow_type == NF9_FTYPE_OPTION) {
     for (num_set = 0; set_event[num_set].type; num_set++)
       (*set_event[num_set].handler)(cache_elem, idata, num_set, &ptr_set, NULL);
   }
@@ -330,7 +330,7 @@ int PG_cache_dbop(struct DBdesc *db, struct db_cache *cache_elem, struct insert_
 
   if (config.sql_dont_try_update || !num_set || (!PG_affected_rows(ret))) {
     /* UPDATE failed, trying with an INSERT query */ 
-    if (cache_elem->flow_type == NF9_FTYPE_EVENT) {
+    if (cache_elem->flow_type == NF9_FTYPE_EVENT || cache_elem->flow_type == NF9_FTYPE_OPTION) {
       strncpy(insert_full_clause, insert_clause, SPACELEFT(insert_full_clause));
       strncat(insert_full_clause, insert_nocounters_clause, SPACELEFT(insert_full_clause));
       strncat(ptr_values, ")", SPACELEFT(values_clause));

src/plugin_common.c

@@ -762,6 +762,10 @@ int P_test_zero_elem(struct chained_cache *elem)
       if (elem->pnat && elem->pnat->nat_event) return FALSE;
       else return TRUE;
     }
+    else if (elem->flow_type == NF9_FTYPE_OPTION) {
+      /* not really much we can test */
+      return FALSE;
+    }
     else {
       if (elem->bytes_counter || elem->packet_counter || elem->flow_counter) return FALSE;
       else return TRUE;

src/pmacct-build.h

@@ -1 +1 @@
-#define PMACCT_BUILD	"20140601-00"
+#define PMACCT_BUILD	"20140602-00"

src/pmacct-data.h

@@ -428,6 +428,7 @@ static const struct _dictionary_line dictionary[] = {
   {"nfacctd_peer_as", cfg_key_nfprobe_peer_as},
   {"nfacctd_pipe_size", cfg_key_nfacctd_pipe_size},
   {"nfacctd_pro_rating", cfg_key_nfacctd_pro_rating},
+  {"nfacctd_account_options", cfg_key_nfacctd_account_options},
   {"pmacctd_proc_name", cfg_key_proc_name},
   {"pmacctd_force_frag_handling", cfg_key_pmacctd_force_frag_handling},
   {"pmacctd_frag_buffer_size", cfg_key_pmacctd_frag_buffer_size},

src/pmacct-defines.h

@@ -388,6 +388,7 @@ typedef u_int32_t pm_counter_t;
 #define NF9_FTYPE_VLAN_MPLS_IPV6        17
 #define NF9_FTYPE_EVENT			100 /* temporary: re-coding needed */
 #define NF9_FTYPE_NAT_EVENT             100
+#define NF9_FTYPE_OPTION		200
 
 /* Packet pointers indexes */
 #define CUSTOM_PRIMITIVE_PACKET_PTR	0

src/pmacct.c

@@ -3294,7 +3294,7 @@ char *pmc_compose_json(u_int64_t wtc, u_int64_t wtc_2, u_int8_t flow_type, struc
     }
   }
 
-  if (flow_type != NF9_FTYPE_EVENT) {
+  if (flow_type != NF9_FTYPE_EVENT && flow_type != NF9_FTYPE_OPTION) {
     kv = json_pack("{sI}", "packets", packet_counter);
     json_object_update_missing(obj, kv);
     json_decref(kv);

src/pretag_handlers.c

@@ -585,6 +585,8 @@ int PT_map_sample_type_handler(char *filename, struct id_entry *e, char *value,
       e->sample_type.n = NF9_FTYPE_TRAFFIC;
     else if (!strncmp(value, "event", strlen("event")))
       e->sample_type.n = NF9_FTYPE_EVENT;
+    else if (!strncmp(value, "option", strlen("option")))
+      e->sample_type.n = NF9_FTYPE_OPTION;
     else {
       Log(LOG_WARNING, "WARN ( %s ): Invalid 'sample_type' value. ", filename);
       return TRUE;

src/sqlite3_plugin.c

@@ -249,7 +249,7 @@ int SQLI_cache_dbop(struct DBdesc *db, struct db_cache *cache_elem, struct inser
   for (num = 0; num < idata->num_primitives; num++)
     (*where[num].handler)(cache_elem, idata, num, &ptr_values, &ptr_where);
 
-  if (cache_elem->flow_type == NF9_FTYPE_EVENT) {
+  if (cache_elem->flow_type == NF9_FTYPE_EVENT || cache_elem->flow_type == NF9_FTYPE_OPTION) {
     for (num_set = 0; set_event[num_set].type; num_set++)
       (*set_event[num_set].handler)(cache_elem, idata, num_set, &ptr_set, NULL);
   }
@@ -271,7 +271,7 @@ int SQLI_cache_dbop(struct DBdesc *db, struct db_cache *cache_elem, struct inser
 
   if (config.sql_dont_try_update || !num_set || (sqlite3_changes(db->desc) == 0)) {
     /* UPDATE failed, trying with an INSERT query */ 
-    if (cache_elem->flow_type == NF9_FTYPE_EVENT) {
+    if (cache_elem->flow_type == NF9_FTYPE_EVENT || cache_elem->flow_type == NF9_FTYPE_OPTION) {
       strncpy(insert_full_clause, insert_clause, SPACELEFT(insert_full_clause));
       strncat(insert_full_clause, insert_nocounters_clause, SPACELEFT(insert_full_clause));
       strncat(ptr_values, ")", SPACELEFT(values_clause));

src/util.c

@@ -2045,7 +2045,7 @@ char *compose_json(u_int64_t wtc, u_int64_t wtc_2, u_int8_t flow_type, struct pk
     json_decref(kv);
   }
 
-  if (flow_type != NF9_FTYPE_EVENT) {
+  if (flow_type != NF9_FTYPE_EVENT && flow_type != NF9_FTYPE_OPTION) {
     kv = json_pack("{sI}", "packets", packet_counter);
     json_object_update_missing(obj, kv);
     json_decref(kv);