stefan
/
tpcpr
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fully implemented linktype rawip

This commit is contained in:
gurkenhabicht 2020-06-13 03:17:13 +02:00
parent 09c8a7438f
commit 4bfd00ac05
5 changed files with 297 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
src/configure/mod.rs
View File

@ -2,10 +2,10 @@
// But at this point of development it seems like this overhead is unjust. // But at this point of development it seems like this overhead is unjust.
extern crate serde_json; extern crate serde_json;
use std::fs::File;
use byteorder::{ByteOrder, LittleEndian}; use byteorder::{ByteOrder, LittleEndian};
use std::collections::HashMap; use std::collections::HashMap;
use std::fs; use std::fs;
use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
//use std::thread::{spawn, JoinHandle}; //use std::thread::{spawn, JoinHandle};
//use std::sync::mpsc::{channel, Receiver}; //use std::sync::mpsc::{channel, Receiver};
@ -28,28 +28,31 @@ pub struct Config {
#[derive(Debug)] #[derive(Debug)]
pub struct FileInfo { pub struct FileInfo {
pub encapsulation_type: u16, pub encapsulation_type: u16,
pub file_size: u64, pub file_size: u64,
pub metadata: std::fs::Metadata, pub metadata: std::fs::Metadata,
} }
impl FileInfo { impl FileInfo {
fn new(file: std::path::PathBuf, encapsulation_type: u16) -> FileInfo { fn new(file: std::path::PathBuf, encapsulation_type: u16) -> FileInfo {
FileInfo { FileInfo {
encapsulation_type: encapsulation_type, encapsulation_type: encapsulation_type,
file_size: fs::metadata(&file).unwrap().len(), file_size: fs::metadata(&file).unwrap().len(),
metadata: fs::metadata(&file).unwrap(), metadata: fs::metadata(&file).unwrap(),
} }
} }
} }
pub fn from_json_file() -> Option<Config> { pub fn from_json_file() -> Option<Config> {
let config_file = File::open("parser.json").expect("file should open read only"); let config_file = File::open("parser.json").expect("file should open read only");
let json: serde_json::Value = serde_json::from_reader(config_file).unwrap(); let json: serde_json::Value = serde_json::from_reader(config_file).unwrap();
Some(Config { Some(Config {
filter: json.get("filter").unwrap().as_str().unwrap().to_owned(), filter: json.get("filter").unwrap().as_str().unwrap().to_owned(),
regex_filter: json.get("regex_filter").unwrap().as_str().unwrap().to_owned(), regex_filter: json
.get("regex_filter")
.unwrap()
.as_str()
.unwrap()
.to_owned(),
insert_max: json.get("insert_max").unwrap().as_u64().unwrap() as usize, insert_max: json.get("insert_max").unwrap().as_u64().unwrap() as usize,
pcap_file: json.get("pcap_file").unwrap().as_str().unwrap().to_owned(), pcap_file: json.get("pcap_file").unwrap().as_str().unwrap().to_owned(),
connection: format!( connection: format!(
@ -69,25 +72,27 @@ pub fn from_json_file() -> Option<Config> {
}) })
} }
/* /*
File signature and encapsulation type from file File signature and encapsulation type from file
See: https://www.tcpdump.org/linktypes.html See: https://www.tcpdump.org/linktypes.html
*/ */
// Futher:file.len() is included in metadata() but only shows up if called explicitly. Maybe this is not needed at all in the end // Futher:file.len() is included in metadata() but only shows up if called explicitly. Maybe this is not needed at all in the end
// This would be needed for comparability over time. print metadata and you will see // This would be needed for comparability over time. print metadata and you will see
fn bytes_from_file( entry: std::path::PathBuf ) -> Result<([u8;4], u16, u16), std::io::Error> { fn bytes_from_file(entry: std::path::PathBuf) -> Result<([u8; 4], u16, u16), std::io::Error> {
let mut magic_number: [u8;4] = [0;4]; let mut magic_number: [u8; 4] = [0; 4];
let mut buffer: [u8;32] = [0;32]; let mut buffer: [u8; 32] = [0; 32];
let mut _file = File::open(entry.to_owned())?; let mut _file = File::open(entry.to_owned())?;
_file.read_exact(&mut buffer)?; _file.read_exact(&mut buffer)?;
magic_number.clone_from_slice(&buffer[0..4]); magic_number.clone_from_slice(&buffer[0..4]);
let enc_pcap: u16 = LittleEndian::read_u16(&buffer[20..22]); let enc_pcap: u16 = LittleEndian::read_u16(&buffer[20..22]);
let enc_pcapng: u16 = LittleEndian::read_u16(&buffer[12..14]); let enc_pcapng: u16 = LittleEndian::read_u16(&buffer[12..14]);
Ok((magic_number, enc_pcap, enc_pcapng)) Ok((magic_number, enc_pcap, enc_pcapng))
} }
pub fn map_pcap_dir ( pcap_dir: &str ) -> Option<std::collections::HashMap<std::path::PathBuf, FileInfo>> { pub fn map_pcap_dir(
pcap_dir: &str,
) -> Option<std::collections::HashMap<std::path::PathBuf, FileInfo>> {
// Well, this conditional intermezzo seems to be best practice. See std::fs doc // Well, this conditional intermezzo seems to be best practice. See std::fs doc
let mut pcap_map = HashMap::new(); let mut pcap_map = HashMap::new();
if let Ok(entries) = fs::read_dir(pcap_dir) { if let Ok(entries) = fs::read_dir(pcap_dir) {
@ -95,10 +100,14 @@ pub fn map_pcap_dir ( pcap_dir: &str ) -> Option<std::collections::HashMap<std::
if let Ok(entry) = entry { if let Ok(entry) = entry {
if let Ok(_file_type) = entry.file_type() { if let Ok(_file_type) = entry.file_type() {
if entry.metadata().unwrap().is_file() { if entry.metadata().unwrap().is_file() {
let (magic_number, enc_pcap, enc_pcapng) = bytes_from_file(entry.path()).unwrap(); let (magic_number, enc_pcap, enc_pcapng) =
bytes_from_file(entry.path()).unwrap();
match magic_number { match magic_number {
PCAPNG_SIGNATURE => pcap_map.insert(entry.path(), FileInfo::new(entry.path(), enc_pcapng) ), PCAPNG_SIGNATURE => pcap_map
PCAP_SIGNATURE | PCAP_SIGNATURE_BE => pcap_map.insert(entry.path(), FileInfo::new(entry.path(), enc_pcap)), // TEST: Endiannes for SIGNATURE_BE may be incorrect after introducing fn bytes_from_file() .insert(entry.path(), FileInfo::new(entry.path(), enc_pcapng)),
PCAP_SIGNATURE | PCAP_SIGNATURE_BE => {
pcap_map.insert(entry.path(), FileInfo::new(entry.path(), enc_pcap))
} // TEST: Endiannes for SIGNATURE_BE may be incorrect after introducing fn bytes_from_file()
_ => None, _ => None,
}; };
// println!("{:?}", &entry.metadata().unwrap().modified()); // println!("{:?}", &entry.metadata().unwrap().modified());
@ -111,5 +120,3 @@ pub fn map_pcap_dir ( pcap_dir: &str ) -> Option<std::collections::HashMap<std::
} }
Some(pcap_map) Some(pcap_map)
} }

143
src/main.rs
View File

@ -8,9 +8,7 @@ mod serializer;
use tokio_postgres::types::ToSql; use tokio_postgres::types::ToSql;
use tokio_postgres::{Error, NoTls}; use tokio_postgres::{Error, NoTls};
fn query_string(insert_max: &usize) -> String { fn query_string(insert_max: &usize) -> String {
let mut insert_template = String::with_capacity(insert_max * 8 + 43); let mut insert_template = String::with_capacity(insert_max * 8 + 43);
insert_template.push_str("INSERT INTO json_dump (packet) Values "); insert_template.push_str("INSERT INTO json_dump (packet) Values ");
@ -30,11 +28,11 @@ async fn main() -> Result<(), Error> {
// TODO: hash file metadata, so its state is comparable at times and can be written to a db table (e.g. after system crash) // TODO: hash file metadata, so its state is comparable at times and can be written to a db table (e.g. after system crash)
// This db table should include UUIDs so it can be joined effectively // This db table should include UUIDs so it can be joined effectively
let pcap_map = configure::map_pcap_dir( &config.pcap_dir ).unwrap(); let pcap_map = configure::map_pcap_dir(&config.pcap_dir).unwrap();
println!("{:?}", pcap_map.iter()); println!("{:?}", pcap_map.iter());
/* db connection */ /* db connection */
let (client, connection) = tokio_postgres::connect(&config.connection, NoTls).await?; let (client, connection) = tokio_postgres::connect(&config.connection, NoTls).await?;
tokio::spawn(async move { tokio::spawn(async move {
@ -54,75 +52,86 @@ async fn main() -> Result<(), Error> {
.await?; .await?;
/* device or file input */ /* device or file input */
match config.is_device { match config.is_device {
false => for (_pcap_file, _pcap_info) in pcap_map.iter() { false => {
println!("{:?}",&_pcap_file); for (_pcap_file, _pcap_info) in pcap_map.iter() {
// TODO: Tuning vector capacity according to mean average & std dev of packet size println!("{:?}", &_pcap_file);
let v: Vec<parser::QryData> = parser::parse(&_pcap_file, &config.filter, &config.regex_filter); // TODO: Tuning vector capacity according to mean average & std dev of packet size
//let mut v = Vec::<parser::QryData>::with_capacity(35536); let v: Vec<parser::QryData> =
//v.extend(parser::parse(&_pcap_file, &config.filter)); parser::parse(&_pcap_file, &config.filter, &config.regex_filter);
//let mut v = Vec::<parser::QryData>::with_capacity(35536);
//v.extend(parser::parse(&_pcap_file, &config.filter));
let packets_serialized = serializer::serialize_packets(v); let packets_serialized = serializer::serialize_packets(v);
//let mut packets_serialized = Vec::<serde_json::Value>::with_capacity(35536); //let mut packets_serialized = Vec::<serde_json::Value>::with_capacity(35536);
//packets_serialized.extend(serializer::serialize_packets(v)); //packets_serialized.extend(serializer::serialize_packets(v));
/* Query */ /* Query */
let chunk_count = packets_serialized.len() / config.insert_max; let chunk_count = packets_serialized.len() / config.insert_max;
let remainder: usize = packets_serialized.len() % config.insert_max; let remainder: usize = packets_serialized.len() % config.insert_max;
let chunker = &packets_serialized.len() < &config.insert_max; let chunker = &packets_serialized.len() < &config.insert_max;
match chunker { match chunker {
true => { true => {
let insert_str = query_string(&packets_serialized.len()); let insert_str = query_string(&packets_serialized.len());
let statement_false = client.prepare(&insert_str).await?; let statement_false = client.prepare(&insert_str).await?;
client
.query_raw(
&statement_false,
packets_serialized.iter().map(|p| p as &dyn ToSql),
)
.await?;
}
false => {
let insert_str = query_string(&config.insert_max);
let statement = client.prepare(&insert_str).await?;
for _i in 0..chunk_count {
let (_input, _) = packets_serialized.split_at(config.insert_max);
client
.query_raw(
&statement,
_input.to_vec().iter().map(|p| p as &dyn ToSql),
)
.await?;
}
if remainder > 0 {
let rem_str = query_string(&remainder);
let statement_remainder = client.prepare(&rem_str).await?;
let (_garbage, _input) =
packets_serialized.split_at(packets_serialized.len() - remainder);
client
.query_raw(
&statement_remainder,
_input.to_vec().iter().map(|p| p as &dyn ToSql),
)
.await?;
}
}
}
}
}
true => {
let insert_str = query_string(&config.insert_max);
let statement = client.prepare(&insert_str).await?;
loop {
let v: Vec<parser::QryData> = parser::parse_device(
&config.device,
&config.filter,
&config.insert_max,
&config.regex_filter,
);
let packets_serialized = serializer::serialize_packets(v);
client client
.query_raw( .query_raw(
&statement_false, &statement,
packets_serialized.iter().map(|p| p as &dyn ToSql), packets_serialized.iter().map(|p| p as &dyn ToSql),
) )
.await?; .await?;
} }
false => {
let insert_str = query_string(&config.insert_max);
let statement = client.prepare(&insert_str).await?;
for _i in 0..chunk_count {
let (_input, _) = packets_serialized.split_at(config.insert_max);
client
.query_raw(&statement, _input.to_vec().iter().map(|p| p as &dyn ToSql))
.await?;
}
if remainder > 0 {
let rem_str = query_string(&remainder);
let statement_remainder = client.prepare(&rem_str).await?;
let (_garbage, _input) =
packets_serialized.split_at(packets_serialized.len() - remainder);
client
.query_raw(
&statement_remainder,
_input.to_vec().iter().map(|p| p as &dyn ToSql),
)
.await?;
}
}
} }
}, }
true => { Ok(())
let insert_str = query_string(&config.insert_max);
let statement = client.prepare(&insert_str).await?;
loop {
let v: Vec<parser::QryData> = parser::parse_device(&config.device, &config.filter, &config.insert_max, &config.regex_filter);
let packets_serialized = serializer::serialize_packets(v);
client
.query_raw(
&statement,
packets_serialized.iter().map(|p| p as &dyn ToSql),
)
.await?;
}
},
}
Ok(())
} }

252
src/parser/mod.rs
View File

@ -4,29 +4,29 @@ extern crate eui48;
mod packet_handler; mod packet_handler;
use pcap::{Capture, Linktype}; use pcap::{Capture, Linktype};
use regex::bytes::Regex; use regex::bytes::Regex;
use std::str;
use std::convert::TryInto; use std::convert::TryInto;
use std::str;
//use std::thread::{spawn, JoinHandle}; //use std::thread::{spawn, JoinHandle};
//use std::sync::mpsc::{channel, Receiver}; //use std::sync::mpsc::{channel, Receiver};
/* protocol ids, LittleEndian */ /* protocol ids, LittleEndian */
const ETH_P_IPV6: usize = 0xDD86; const ETH_P_IPV6: usize = 0xDD86;
const ETH_P_IP: usize = 0x08; const ETH_P_IP: usize = 0x08;
const TCP: usize = 0x06; const TCP: usize = 0x06;
const UDP: usize = 0x11; const UDP: usize = 0x11;
const ETH_P_ARP: usize = 0x0608; const ETH_P_ARP: usize = 0x0608;
const ETH_P_RARP: usize = 0x3580; const ETH_P_RARP: usize = 0x3580;
const ETHER_HDRLEN: usize = 14;
/* /*
QryData could be written in the sense of QryData{ ... frame: .., packet: .., segment:.. } QryData could be written in the sense of QryData{ ... frame: .., packet: .., segment:.. }
On the one hand, only the actual type of frame/packet/segment would be contained in the resulting struct. On the one hand, only the actual type of frame/packet/segment would be contained in the resulting struct.
So, increased benefit in serialization/cpu time, could result in less data to be serialized, depending on layout. So, increased benefit in serialization/cpu time, could result in less data to be serialized, depending on layout.
On the other hand, each datagram::type needs to implement traits which would need to be dynamically dispatched by returning any of these types per iso level from a single function each. The result would be a performance decrease. On the other hand, each datagram::type needs to implement traits which would need to be dynamically dispatched by returning any of these types per iso level from a single function each. The result would be a performance decrease.
See: https://doc.rust-lang.org/book/ch10-02-traits.html#returning-types-that-implement-traits See: https://doc.rust-lang.org/book/ch10-02-traits.html#returning-types-that-implement-traits
See: https://doc.rust-lang.org/book/ch17-02-trait-objects.html#trait-objects-perform-dynamic-dispatch See: https://doc.rust-lang.org/book/ch17-02-trait-objects.html#trait-objects-perform-dynamic-dispatch
Then again, parser logic would be fewer lines + more unified using the latter method. Maybe better optimizable as well? Maybe this is a nice tradeoff? Then again, parser logic would be fewer lines + more unified using the latter method. Maybe better optimizable as well? Maybe this is a nice tradeoff?
TODO: Implement and benchmark dynamically dispatched packet data in conjunction with restructured QryData. TODO: Implement and benchmark dynamically dispatched packet data in conjunction with restructured QryData.
*/ */
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct QryData { pub struct QryData {
@ -42,115 +42,120 @@ pub struct QryData {
pub reg_res: Option<String>, pub reg_res: Option<String>,
} }
fn init_qrydata( ) -> Result<QryData, core::fmt::Error> { #[allow(dead_code)]
Ok(QryData { enum EncapsulationType {
id: 0, // pcap::Linktype::get_name() is unsafe.
time: 0.0, EN10MB = 1, // See: https://docs.rs/pcap/0.7.0/src/pcap/lib.rs.html#247-261
data: None, RAW = 101, // Would this be an issue?
ether_header: None::<packet_handler::EtherHeader>,
ipv4_header: None::<packet_handler::IpV4Header>,
ipv6_header: None::<packet_handler::IpV6Header>,
tcp_header: None::<packet_handler::TcpHeader>,
udp_header: None::<packet_handler::UdpHeader>,
arp_header: None::<packet_handler::ArpHeader>,
reg_res: None::<String>,
})
} }
#[allow(dead_code)]
enum EncapsulationType { // pcap::Linktype::get_name() is unsafe.
EN10MB = 1, // See: https://docs.rs/pcap/0.7.0/src/pcap/lib.rs.html#247-261
RAW = 101, // Would this be an issue?
}
impl QryData { impl QryData {
// This is not cool! // This is not cool!
// This will get modularized into subfunctions // Implementing objectoriented is slower by 3-10%. Variance is all over the place. It's awful but modular!
fn encap_en10mb( packet_data: &[u8] ) -> Option<QryData> { // Guess I'll do a roolback and do a different design
let mut pkg: QryData = init_qrydata().unwrap();
pkg.ether_header = Some(packet_handler::ethernet_handler(packet_data)).unwrap(); fn new() -> QryData {
match pkg.ether_header.unwrap().ether_type as usize { QryData {
id: 0,
time: 0.0,
data: None,
ether_header: None::<packet_handler::EtherHeader>,
ipv4_header: None::<packet_handler::IpV4Header>,
ipv6_header: None::<packet_handler::IpV6Header>,
tcp_header: None::<packet_handler::TcpHeader>,
udp_header: None::<packet_handler::UdpHeader>,
arp_header: None::<packet_handler::ArpHeader>,
reg_res: None::<String>,
}
}
fn encap_en10mb(&mut self, packet_data: &[u8]) -> Result<(), core::fmt::Error> {
//let mut pkg: QryData = new().unwrap();
self.ether_header = Some(packet_handler::ethernet_handler(packet_data)).unwrap();
match self.ether_header.unwrap().ether_type as usize {
ETH_P_IP => { ETH_P_IP => {
pkg.ipv4_header = Some(packet_handler::ip_handler(packet_data)).unwrap(); self.ipv4_header = Some(packet_handler::ip_handler(packet_data, ETHER_HDRLEN)).unwrap();
let protocol_type = pkg.ipv4_header.unwrap().ip_protocol.clone() as usize; let protocol_type = self.ipv4_header.unwrap().ip_protocol as usize;
let l3_header_length = pkg.ipv4_header.unwrap().ip_ihl; let l3_header_length = self.ipv4_header.unwrap().ip_ihl;
pkg.transport_layer(packet_data, protocol_type, l3_header_length).unwrap(); self.transport_layer(packet_data, protocol_type, l3_header_length, ETHER_HDRLEN)
.unwrap();
} }
ETH_P_IPV6 => { ETH_P_IPV6 => {
pkg.ipv6_header = Some(packet_handler::ipv6_handler(packet_data)).unwrap(); self.ipv6_header = Some(packet_handler::ipv6_handler(packet_data, ETHER_HDRLEN)).unwrap();
let protocol_type = pkg.ipv6_header.unwrap().next_header.clone() as usize; let protocol_type = self.ipv6_header.unwrap().next_header as usize;
pkg.transport_layer(packet_data, protocol_type, 10).unwrap(); self.transport_layer(packet_data, protocol_type, 10, ETHER_HDRLEN)
.unwrap();
} }
ETH_P_ARP | ETH_P_RARP => { ETH_P_ARP | ETH_P_RARP => {
pkg.arp_header = Some(packet_handler::arp_handler(packet_data)).unwrap(); self.arp_header = Some(packet_handler::arp_handler(packet_data, ETHER_HDRLEN)).unwrap();
} }
_ => println!("Network protocol not implemented"), _ => println!("Network protocol not implemented"),
} }
Some(pkg) Ok(())
} }
fn encap_raw ( packet_data: &[u8] ) -> Option<QryData> { fn encap_raw(&mut self, packet_data: &[u8]) -> Result<(), core::fmt::Error> {
let mut pkg: QryData = init_qrydata().unwrap(); // let mut pkg: QryData = new().unwrap();
let ip_version: usize = ((packet_data[0] & 0xf0) >> 4).try_into().unwrap(); let ip_version: usize = ((packet_data[0] & 0xf0) >> 4).try_into().unwrap();
match ip_version { //println!("{:?}", &ip_version);
4 => { match ip_version {
pkg.ipv4_header = Some(packet_handler::ip_handler(packet_data)).unwrap(); 4 => {
let protocol_type = pkg.ipv4_header.unwrap().ip_protocol.clone() as usize; //println!("v4");
let l3_header_length = pkg.ipv4_header.unwrap().ip_ihl; self.ipv4_header = Some(packet_handler::ip_handler(packet_data, 0)).unwrap();
pkg.transport_layer(packet_data, protocol_type, l3_header_length).unwrap(); let protocol_type = self.ipv4_header.unwrap().ip_protocol as usize;
} let l3_header_length = self.ipv4_header.unwrap().ip_ihl;
6 => { self.transport_layer(packet_data, protocol_type, l3_header_length, 0)
pkg.ipv6_header = Some(packet_handler::ipv6_handler(packet_data)).unwrap(); .unwrap();
let protocol_type = pkg.ipv6_header.unwrap().next_header.clone() as usize; }
pkg.transport_layer(packet_data, protocol_type, 10).unwrap(); 6 => {
} //println!("v6");
_ => println!("Network Protocol not implemented") self.ipv6_header = Some(packet_handler::ipv6_handler(packet_data, 0)).unwrap();
} let protocol_type = self.ipv6_header.unwrap().next_header as usize;
Some(pkg) self.transport_layer(packet_data, protocol_type, 10, 0)
.unwrap();
}
_ => println!("Network Protocol not implemented"),
}
Ok(())
} }
// TODO: impl correct Err type and use in Result // TODO: impl correct Err type and use in Result
fn transport_layer (&mut self, packet_data: &[u8], protocol_type: usize, l3_header_length: u32) -> Result<(), core::fmt::Error> { fn transport_layer(
match protocol_type { &mut self,
TCP => { packet_data: &[u8],
protocol_type: usize,
self.tcp_header = Some(packet_handler::tcp_handler( l3_header_length: u32,
l3_header_length, ether_hdrlen: usize,
packet_data, ) -> Result<(), core::fmt::Error> {
match protocol_type {
)) TCP => {
.unwrap(); self.tcp_header =
self.data = Some(packet_handler::payload_handler( Some(packet_handler::tcp_handler(l3_header_length, packet_data, ether_hdrlen)).unwrap();
l3_header_length, self.data = Some(packet_handler::payload_handler(
self.tcp_header.unwrap().data_offset, l3_header_length,
packet_data, self.tcp_header.unwrap().data_offset,
)).unwrap(); packet_data,
} ether_hdrlen
UDP => { ))
.unwrap();
self.udp_header = Some(packet_handler::udp_handler( }
l3_header_length, UDP => {
packet_data, self.udp_header =
)) Some(packet_handler::udp_handler(l3_header_length, packet_data, ether_hdrlen)).unwrap();
.unwrap(); self.data = Some(packet_handler::payload_handler(
self.data = Some(packet_handler::payload_handler( l3_header_length,
l3_header_length, 7,
7, packet_data,
packet_data, ether_hdrlen
)).unwrap(); ))
} .unwrap();
_ => println!("Transport layer protocol not implemented"), }
} _ => println!("Transport layer protocol not implemented"),
}
Ok(()) Ok(())
} }
} }
/* Regex parse _complete_ package */ /* Regex parse _complete_ package */
fn flag_carnage(re: &Regex, payload: &[u8]) -> Option<String> { fn flag_carnage(re: &Regex, payload: &[u8]) -> Option<String> {
let mut flags: String = String::new(); let mut flags: String = String::new();
@ -168,7 +173,7 @@ fn flag_carnage(re: &Regex, payload: &[u8]) -> Option<String> {
} }
pub fn parse(parse_file: &std::path::Path, filter_str: &str, regex_filter: &str) -> Vec<QryData> { pub fn parse(parse_file: &std::path::Path, filter_str: &str, regex_filter: &str) -> Vec<QryData> {
let mut me: QryData = init_qrydata().unwrap(); //let mut me: QryData = QryData::new();
let mut v: Vec<QryData> = Vec::new(); let mut v: Vec<QryData> = Vec::new();
let mut cap = Capture::from_file(parse_file).unwrap(); let mut cap = Capture::from_file(parse_file).unwrap();
@ -177,43 +182,58 @@ pub fn parse(parse_file: &std::path::Path, filter_str: &str, regex_filter: &str)
println!("{:?}", &linktype); println!("{:?}", &linktype);
let re = Regex::new(regex_filter).unwrap(); let re = Regex::new(regex_filter).unwrap();
while let Ok(packet) = cap.next() { while let Ok(packet) = cap.next() {
let mut me = QryData::new();
match linktype { match linktype {
// Syntax is clunky, but no num_derive + num_traits dependencies. Linktype(1) => me.encap_en10mb(packet.data).unwrap(), //me = QryData::encap_en10mb(packet.data).unwrap(), // EN10MB
Linktype(1) => me = QryData::encap_en10mb(packet.data).unwrap(), // EN10MB Linktype(12) => me.encap_raw(packet.data).unwrap(), //me = QryData::encap_raw(packet.data).unwrap(), // RAW
Linktype(101) => me = QryData::encap_raw(packet.data).unwrap(), // RAW
_ => (), _ => (),
}; };
me.time = (packet.header.ts.tv_usec as f64 / 1000000.0) + packet.header.ts.tv_sec as f64; me.time = (packet.header.ts.tv_usec as f64 / 1000000.0) + packet.header.ts.tv_sec as f64;
//me.data = Some(packet.data.to_vec());
me.reg_res = flag_carnage(&re, packet.data); // Regex overhead is between 4-9% --single threaded-- on complete packet [u8] data me.reg_res = flag_carnage(&re, packet.data); // Regex overhead is between 4-9% --single threaded-- on complete packet [u8] data
v.push(me.clone()); //v.push(me.clone());
v.push(QryData {
id: 0,
time: me.time,
data: me.data,
ether_header: me.ether_header,
ipv4_header: me.ipv4_header,
ipv6_header: me.ipv6_header,
tcp_header: me.tcp_header,
udp_header: me.udp_header,
arp_header: me.arp_header,
reg_res: me.reg_res,
});
} }
v v
} }
/* This could need some love */ /* This could need some love */
pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize, regex_filter: &str) -> Vec<QryData> { pub fn parse_device(
let mut me: QryData = init_qrydata ( ).unwrap(); parse_device: &str,
filter_str: &str,
insert_max: &usize,
regex_filter: &str,
) -> Vec<QryData> {
//let mut me: QryData = QryData::new ( );
let mut v: Vec<QryData> = Vec::new(); let mut v: Vec<QryData> = Vec::new();
let mut cap = Capture::from_device(parse_device).unwrap().open().unwrap(); let mut cap = Capture::from_device(parse_device).unwrap().open().unwrap();
Capture::filter(&mut cap, &filter_str).unwrap(); Capture::filter(&mut cap, &filter_str).unwrap();
let linktype = cap.get_datalink(); let linktype = cap.get_datalink();
let re = Regex::new(regex_filter).unwrap(); let re = Regex::new(regex_filter).unwrap();
'parse: while let Ok(packet) = cap.next() { 'parse: while let Ok(packet) = cap.next() {
match linktype { let mut me = QryData::new();
Linktype(1) => me = QryData::encap_en10mb(packet.data).unwrap(), match linktype {
Linktype(101) => me = QryData::encap_raw(packet.data).unwrap(), Linktype(1) => me.encap_en10mb(packet.data).unwrap(), //me = QryData::encap_en10mb(packet.data).unwrap(),
Linktype(101) => me.encap_raw(packet.data).unwrap(), //me = QryData::encap_raw(packet.data).unwrap(),
_ => (), _ => (),
} };
me.time = (packet.header.ts.tv_usec as f64 / 1000000.0) + packet.header.ts.tv_sec as f64; me.time = (packet.header.ts.tv_usec as f64 / 1000000.0) + packet.header.ts.tv_sec as f64;
me.reg_res = flag_carnage(&re, packet.data); me.reg_res = flag_carnage(&re, packet.data);
v.push(me.clone()); v.push(me.clone());
if &v.len() >= insert_max { if &v.len() >= insert_max {
break 'parse; break 'parse;
} }

73
src/parser/packet_handler.rs
View File

@ -1,7 +1,7 @@
extern crate bitfield; extern crate bitfield;
extern crate byteorder; extern crate byteorder;
extern crate eui48; extern crate eui48;
extern crate serde; pub extern crate serde;
use bitfield::bitfield; use bitfield::bitfield;
use byteorder::{BigEndian, ByteOrder, LittleEndian}; use byteorder::{BigEndian, ByteOrder, LittleEndian};
use eui48::{Eui48, MacAddress}; use eui48::{Eui48, MacAddress};
@ -13,14 +13,13 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
/* ethernet */ /* ethernet */
const ETH_ALEN: usize = 6; const ETH_ALEN: usize = 6;
const ETH_TLEN: usize = 2; const ETH_TLEN: usize = 2;
const ETHER_HDRLEN: usize = 14; //const ETHER_HDRLEN: usize = 14;
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy,Serialize, Deserialize)]
pub struct EtherHeader { pub struct EtherHeader {
pub ether_dhost: MacAddress, pub ether_dhost: MacAddress,
pub ether_shost: MacAddress, pub ether_shost: MacAddress,
pub ether_type: i32, pub ether_type: i32,
} }
// TODO: implement optional ethernet vlan shim header fields // TODO: implement optional ethernet vlan shim header fields
@ -89,9 +88,9 @@ impl<T: AsRef<[u8]> + AsMut<[u8]>> BitfieldIpV4Header<T> {
} }
} }
pub fn ip_handler(packet_data: &[u8]) -> Option<IpV4Header> { pub fn ip_handler(packet_data: &[u8], ether_hdrlen: usize) -> Option<IpV4Header> {
let mut raw_hdr: [u8; 20] = [0; 20]; let mut raw_hdr: [u8; 20] = [0; 20];
raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN..ETHER_HDRLEN + 20]); raw_hdr.copy_from_slice(&packet_data[ether_hdrlen..ether_hdrlen + 20]);
let ip_header = BitfieldIpV4Header(raw_hdr); let ip_header = BitfieldIpV4Header(raw_hdr);
Some(IpV4Header { Some(IpV4Header {
@ -125,9 +124,9 @@ pub struct IpV6Header {
pub destination_address: IpAddr, pub destination_address: IpAddr,
} }
pub fn ipv6_handler(packet_data: &[u8]) -> Option<IpV6Header> { pub fn ipv6_handler(packet_data: &[u8], ether_hdrlen: usize) -> Option<IpV6Header> {
let mut raw_hdr: [u8; 40] = [0; 40]; let mut raw_hdr: [u8; 40] = [0; 40];
raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN..ETHER_HDRLEN + 40]); raw_hdr.copy_from_slice(&packet_data[ether_hdrlen..ether_hdrlen + 40]);
Some(IpV6Header { Some(IpV6Header {
version: (&raw_hdr[0] & 0xf0) >> 4, version: (&raw_hdr[0] & 0xf0) >> 4,
@ -187,10 +186,10 @@ bitfield! {
get_urgent_pointer, _: 159,144; get_urgent_pointer, _: 159,144;
} }
pub fn tcp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<TcpHeader> { pub fn tcp_handler(ip_hlen: u32, packet_data: &[u8], ether_hdrlen: usize) -> Option<TcpHeader> {
let mut raw_hdr: [u8; 20] = [0; 20]; let mut raw_hdr: [u8; 20] = [0; 20];
raw_hdr.copy_from_slice( raw_hdr.copy_from_slice(
&packet_data[ETHER_HDRLEN + ip_hlen as usize * 4..ETHER_HDRLEN + ip_hlen as usize * 4 + 20], &packet_data[ether_hdrlen + ip_hlen as usize * 4..ether_hdrlen+ ip_hlen as usize * 4 + 20],
); );
let tcp_header = BitfieldTcpHeader(raw_hdr); let tcp_header = BitfieldTcpHeader(raw_hdr);
@ -221,40 +220,38 @@ pub fn tcp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<TcpHeader> {
pub struct ArpHeader { pub struct ArpHeader {
pub htype: u16, pub htype: u16,
pub ptype: u16, pub ptype: u16,
pub hlen: u8, pub hlen: u8,
pub plen: u8, pub plen: u8,
pub oper: u16, pub oper: u16,
pub sha: MacAddress, pub sha: MacAddress,
pub spa: IpAddr, pub spa: IpAddr,
pub tha: MacAddress, pub tha: MacAddress,
pub tpa: IpAddr, pub tpa: IpAddr,
} }
pub fn arp_handler(packet_data: &[u8]) -> Option<ArpHeader> { pub fn arp_handler(packet_data: &[u8], ether_hdrlen: usize) -> Option<ArpHeader> {
let mut raw_hdr: [u8; 28] = [0; 28]; let mut raw_hdr: [u8; 28] = [0; 28];
raw_hdr.copy_from_slice( raw_hdr.copy_from_slice(&packet_data[ether_hdrlen..ether_hdrlen + 28]);
&packet_data[ETHER_HDRLEN .. ETHER_HDRLEN + 28]
);
let mut _sha: [u8; 6] = [0; 6]; let mut _sha: [u8; 6] = [0; 6];
let mut _tha: [u8; 6] = [0; 6]; let mut _tha: [u8; 6] = [0; 6];
_sha.copy_from_slice(&raw_hdr[8..14]); _sha.copy_from_slice(&raw_hdr[8..14]);
_tha.copy_from_slice(&raw_hdr[18..24]); _tha.copy_from_slice(&raw_hdr[18..24]);
Some(ArpHeader{ Some(ArpHeader {
htype: BigEndian::read_u16(&raw_hdr[0..2]), htype: BigEndian::read_u16(&raw_hdr[0..2]),
ptype: BigEndian::read_u16(&raw_hdr[2..4]), ptype: BigEndian::read_u16(&raw_hdr[2..4]),
hlen: raw_hdr[4], hlen: raw_hdr[4],
plen: raw_hdr[5], plen: raw_hdr[5],
oper: BigEndian::read_u16(&raw_hdr[6..8]), oper: BigEndian::read_u16(&raw_hdr[6..8]),
sha: MacAddress::new(_sha as Eui48), sha: MacAddress::new(_sha as Eui48),
spa: IpAddr::V4(Ipv4Addr::from(BigEndian::read_u32(&raw_hdr[14..18]))), spa: IpAddr::V4(Ipv4Addr::from(BigEndian::read_u32(&raw_hdr[14..18]))),
tha: MacAddress::new( _tha as Eui48 ), tha: MacAddress::new(_tha as Eui48),
tpa: IpAddr::V4(Ipv4Addr::from(BigEndian::read_u32(&raw_hdr[24..28]))), tpa: IpAddr::V4(Ipv4Addr::from(BigEndian::read_u32(&raw_hdr[24..28]))),
}) })
} }
/* udp */ /* udp */
#[derive(Debug, Copy, Clone, Serialize, Deserialize)] #[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub struct UdpHeader { pub struct UdpHeader {
@ -264,10 +261,10 @@ pub struct UdpHeader {
pub checksum: u16, pub checksum: u16,
} }
pub fn udp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<UdpHeader> { pub fn udp_handler(ip_hlen: u32, packet_data: &[u8], ether_hdrlen: usize) -> Option<UdpHeader> {
let mut raw_hdr: [u8; 8] = [0; 8]; let mut raw_hdr: [u8; 8] = [0; 8];
raw_hdr.copy_from_slice( raw_hdr.copy_from_slice(
&packet_data[ETHER_HDRLEN + ip_hlen as usize * 4..ETHER_HDRLEN + ip_hlen as usize * 4 + 8], &packet_data[ether_hdrlen + ip_hlen as usize * 4..ether_hdrlen + ip_hlen as usize * 4 + 8],
); );
Some(UdpHeader { Some(UdpHeader {
@ -279,8 +276,8 @@ pub fn udp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<UdpHeader> {
} }
/* payload */ /* payload */
pub fn payload_handler(ip_hlen: u32, data_offset: u32, packet_data: &[u8]) -> Option<Vec<u8>> { pub fn payload_handler(ip_hlen: u32, data_offset: u32, packet_data: &[u8], ether_hdrlen: usize) -> Option<Vec<u8>> {
let (_head, tail) = let (_head, tail) =
packet_data.split_at(ETHER_HDRLEN + ip_hlen as usize * 4 + data_offset as usize * 4); packet_data.split_at(ether_hdrlen + ip_hlen as usize * 4 + data_offset as usize * 4);
Some(tail.to_vec()) Some(tail.to_vec())
} }

26
src/serializer/mod.rs
View File

@ -27,29 +27,35 @@ impl Serialize for parser::QryData {
} }
pub fn serialize_packets(v: Vec<parser::QryData>) -> Vec<serde_json::Value> { pub fn serialize_packets(v: Vec<parser::QryData>) -> Vec<serde_json::Value> {
/* rayon parallelized */ /* rayon parallelized */
// TODO: Benchmark. As far as I tested, this reaps no benefit. // TODO: Benchmark. As far as I tested, this reaps no benefit.
let packets_serialized = v.par_iter().map(|x| serde_json::to_value(x).unwrap()).collect(); let packets_serialized = v
//let packets_serialized: Vec<serde_json::Value> = v.par_iter().map(|x| json!(x)).collect(); .par_iter()
.map(|x| serde_json::to_value(x).unwrap())
.collect();
//let packets_serialized: Vec<serde_json::Value> = v.par_iter().map(|x| json!(x)).collect();
packets_serialized packets_serialized
} }
// This is way faster than serialize_packets() but I can not figure out how to properly select parts from the resulting json structure as an sql query
// This is way faster than serialize_packets() but I can not figure out how to properly select parts from the resulting json structure as an sql query
#[allow(dead_code)] #[allow(dead_code)]
pub fn serialize_packets_as_string(v: Vec<parser::QryData>) -> Vec<serde_json::Value> { pub fn serialize_packets_as_string(v: Vec<parser::QryData>) -> Vec<serde_json::Value> {
let mut packets_serialized: Vec<serde_json::Value> = Vec::with_capacity(v.len() * 2); let mut packets_serialized: Vec<serde_json::Value> = Vec::with_capacity(v.len() * 2);
for packet in v.iter() { for packet in v.iter() {
packets_serialized.push(serde_json::Value::String(serde_json::to_string(&packet).unwrap())); packets_serialized.push(serde_json::Value::String(
} serde_json::to_string(&packet).unwrap(),
));
}
packets_serialized packets_serialized
} }
#[allow(dead_code)] #[allow(dead_code)]
fn smallest_prime_divisor(remainder: usize ) -> usize { fn smallest_prime_divisor(remainder: usize) -> usize {
let smallest_divisor: usize = (2..(remainder/2)).into_par_iter().find_first(|x| remainder % x == 0).unwrap(); let smallest_divisor: usize = (2..(remainder / 2))
.into_par_iter()
.find_first(|x| remainder % x == 0)
.unwrap();
smallest_divisor smallest_divisor
} }