diff --git "a/src/\\" "b/src/\\" new file mode 100644 index 0000000..b11b862 --- /dev/null +++ "b/src/\\" @@ -0,0 +1,244 @@ +extern crate bitfield; +extern crate byteorder; +extern crate eui48; +mod packet_handler; +use eui48::MacAddress; +use pcap::Capture; +use regex::bytes::Regex; +use std::str; + +/* protocol ids, LittleEndian */ +const ETH_P_IPV6: usize = 0xDD86; +const ETH_P_IP: usize = 0x08; +const TCP: usize = 0x06; +const UDP: usize = 0x11; +const ETH_P_ARP: usize = 0x0608; +const ETH_P_RARP: usize = 0x3580; + +fn build_ether() -> packet_handler::EtherHeader { + packet_handler::EtherHeader { + ether_dhost: (MacAddress::new([0; 6])).to_hex_string(), + ether_shost: (MacAddress::new([0; 6])).to_hex_string(), + ether_type: 0, + } +} + +#[derive(Debug, Clone)] +pub struct QryData { + pub id: i32, + pub time: f64, + pub data: Option>, + pub ether_header: packet_handler::EtherHeader, + pub ipv4_header: Option, + pub ipv6_header: Option, + pub tcp_header: Option, + pub udp_header: Option, + pub arp_header: Option, + pub reg_res: Option, +} + +fn flag_carnage(re: &Regex, payload: &[u8]) -> Option { + let mut flags: String = String::new(); + for mat in re.find_iter(payload) { + flags.push_str(std::str::from_utf8(mat.as_bytes()).unwrap()); + } + match 0 < flags.len() { + false => None, + true => Some(flags), + } +} + +pub fn parse <'a>(parse_file: &std::path::Path, filter_str: &str) -> Vec { + let ether_init = build_ether(); + + let mut me = QryData { + id: 0, + time: 0.0, + data: None, + ether_header: ether_init, + ipv4_header: None::, + ipv6_header: None::, + tcp_header: None::, + udp_header: None::, + arp_header: None::>, + reg_res: None::, + }; + let mut v: Vec = Vec::new(); + + let mut cap = Capture::from_file(parse_file).unwrap(); + Capture::filter(&mut cap, &filter_str).unwrap(); + let re = Regex::new(r"(?:http|https):[[::punct::]]?").unwrap(); + while let Ok(packet) = cap.next() { + me.time = (packet.header.ts.tv_usec as f64 / 1000000.0) + packet.header.ts.tv_sec as f64; + me.data = Some(packet.data.to_vec()); + me.reg_res = flag_carnage(&re, packet.data); + me.ether_header = packet_handler::ethernet_handler(packet.data); + match me.ether_header.ether_type as usize { + ETH_P_IP => { + me.ipv6_header = None::; + me.ipv4_header = Some(packet_handler::ip_handler(packet.data)).unwrap(); + match me.ipv4_header.unwrap().ip_protocol as usize { + TCP => { + me.tcp_header = Some(packet_handler::tcp_handler( + me.ipv4_header.unwrap().ip_ihl, + packet.data, + )) + .unwrap(); + me.data = packet_handler::payload_handler( + me.ipv4_header.unwrap().ip_ihl, + me.tcp_header.unwrap().data_offset, + packet.data, + ); + } + UDP => { + me.udp_header = Some(packet_handler::udp_handler( + me.ipv4_header.unwrap().ip_ihl, + packet.data, + )) + .unwrap(); + me.data = packet_handler::payload_handler( + me.ipv4_header.unwrap().ip_ihl, + 7, + packet.data, + ); + } + _ => println!("network protocol not implemented"), + } + } + ETH_P_IPV6 => { + me.ipv4_header = None::; + me.ipv6_header = Some(packet_handler::ipv6_handler(packet.data)).unwrap(); + match me.ipv6_header.unwrap().next_header as usize { + TCP => { + me.tcp_header = Some(packet_handler::tcp_handler(10, packet.data)).unwrap(); + me.data = packet_handler::payload_handler( + 10, + me.tcp_header.unwrap().data_offset, + packet.data, + ); + } + UDP => { + me.udp_header = Some(packet_handler::udp_handler(10, packet.data)).unwrap(); + me.data = packet_handler::payload_handler(10, 7, packet.data); + } + _ => println!("network protocol not implemented"), + } + } + ETH_P_ARP | ETH_P_RARP => { + me.arp_header = Some(packet_handler::arp_handler(packet.data)).unwrap(); + } + _ => println!("network protocol not implemented"), + } + v.push(QryData { + id: 0, + time: me.time, + data: me.data, + ether_header: me.ether_header, + ipv4_header: me.ipv4_header, + ipv6_header: me.ipv6_header, + tcp_header: me.tcp_header, + udp_header: me.udp_header, + arp_header: me.arp_header, + reg_res: me.reg_res, + }); + } + v +} + +pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) -> Vec { + let ether_init = build_ether(); + + let mut me = QryData { + id: 0, + time: 0.0, + data: None, + ether_header: ether_init, + ipv4_header: None::, + ipv6_header: None::, + tcp_header: None::, + udp_header: None::, + arp_header: None::, + reg_res: None::, + }; + let mut v: Vec = Vec::new(); + let mut cap = Capture::from_device(parse_device).unwrap().open().unwrap(); + Capture::filter(&mut cap, &filter_str).unwrap(); + + let re = Regex::new(r"(?:http|https):[[::punct::]]").unwrap(); + 'parse: while let Ok(packet) = cap.next() { + me.time = (packet.header.ts.tv_usec as f64 / 1000000.0) + packet.header.ts.tv_sec as f64; + me.data = Some(packet.data.to_vec()); + me.reg_res = flag_carnage(&re, packet.data); + me.ether_header = packet_handler::ethernet_handler(packet.data); + match me.ether_header.ether_type as usize { + ETH_P_IP => { + me.ipv6_header = None::; + me.ipv4_header = Some(packet_handler::ip_handler(packet.data)).unwrap(); + match me.ipv4_header.unwrap().ip_protocol as usize { + TCP => { + me.tcp_header = Some(packet_handler::tcp_handler( + me.ipv4_header.unwrap().ip_ihl, + packet.data, + )) + .unwrap(); + me.data = packet_handler::payload_handler( + me.ipv4_header.unwrap().ip_ihl, + me.tcp_header.unwrap().data_offset, + packet.data, + ); + } + UDP => { + me.udp_header = Some(packet_handler::udp_handler( + me.ipv4_header.unwrap().ip_ihl, + packet.data, + )) + .unwrap(); + me.data = packet_handler::payload_handler( + me.ipv4_header.unwrap().ip_ihl, + 7, + packet.data, + ); + } + _ => println!("network protocol not implemented"), + } + } + ETH_P_IPV6 => { + me.ipv4_header = None::; + me.ipv6_header = Some(packet_handler::ipv6_handler(packet.data)).unwrap(); + match me.ipv6_header.unwrap().next_header as usize { + TCP => { + me.tcp_header = Some(packet_handler::tcp_handler(10, packet.data)).unwrap(); + me.data = packet_handler::payload_handler( + 10, + me.tcp_header.unwrap().data_offset, + packet.data, + ); + } + UDP => { + me.udp_header = Some(packet_handler::udp_handler(10, packet.data)).unwrap(); + me.data = packet_handler::payload_handler(10, 7, packet.data); + } + _ => println!("network protocol not implemented"), + } + } + _ => println!("network protocol not implemented"), + } + v.push(QryData { + id: 0, + time: me.time, + data: me.data, + ether_header: me.ether_header, + ipv4_header: me.ipv4_header, + ipv6_header: me.ipv6_header, + tcp_header: me.tcp_header, + udp_header: me.udp_header, + arp_header: me.arp_header, + reg_res: me.reg_res, + }); + + if &v.len() >= insert_max { + break 'parse; + } + } + v +} diff --git a/src/configure/mod.rs b/src/configure/mod.rs index da70713..55e09ed 100644 --- a/src/configure/mod.rs +++ b/src/configure/mod.rs @@ -1,4 +1,3 @@ - // Init of configuration files could also be done via Config crate. // But at this point of development it seems like unjustified overhead. @@ -10,27 +9,31 @@ pub struct Config { pub insert_max: usize, pub pcap_file: String, pub connection: String, - pub device: String, + pub device: String, pub is_device: bool, pub pcap_dir: String, } - pub fn from_json_file() -> Option { - let config_file = File::open("parser.json").expect("file should open read only"); - let json: serde_json::Value = serde_json::from_reader(config_file).unwrap(); - Some(Config { - filter: json.get("filter").unwrap().as_str().unwrap().to_owned(), - insert_max : json.get("insert_max").unwrap().as_u64().unwrap() as usize, - pcap_file : json.get("pcap_file").unwrap().as_str().unwrap().to_owned(), - connection : format!("host={} user={} password={}", - json.get("database_host").unwrap().as_str().unwrap(), - json.get("database_user").unwrap().as_str().unwrap(), - json.get("database_password").unwrap().as_str().unwrap(), - ), - device : json.get("parse_device").unwrap().as_str().unwrap().to_owned(), - is_device : json.get("from_device").unwrap().as_bool().unwrap(), - pcap_dir : json.get("pcap_dir").unwrap().as_str().unwrap().to_owned(), - }) + let config_file = File::open("parser.json").expect("file should open read only"); + let json: serde_json::Value = serde_json::from_reader(config_file).unwrap(); + Some(Config { + filter: json.get("filter").unwrap().as_str().unwrap().to_owned(), + insert_max: json.get("insert_max").unwrap().as_u64().unwrap() as usize, + pcap_file: json.get("pcap_file").unwrap().as_str().unwrap().to_owned(), + connection: format!( + "host={} user={} password={}", + json.get("database_host").unwrap().as_str().unwrap(), + json.get("database_user").unwrap().as_str().unwrap(), + json.get("database_password").unwrap().as_str().unwrap(), + ), + device: json + .get("parse_device") + .unwrap() + .as_str() + .unwrap() + .to_owned(), + is_device: json.get("from_device").unwrap().as_bool().unwrap(), + pcap_dir: json.get("pcap_dir").unwrap().as_str().unwrap().to_owned(), + }) } - diff --git a/src/main.rs b/src/main.rs index d1d1416..779a656 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,7 @@ const PCAP_SIGNATURE: [u8; 4] = [0xed, 0xab, 0xee, 0xdb]; fn query_string(insert_max: &usize) -> String { - // Changed this to String with given capacity after stumbling on https://github.com/hoodie/concatenation_benchmarks-rs + // Changed this to String with given capacity after stumbling over https://github.com/hoodie/concatenation_benchmarks-rs // Impressive performance increase! let mut insert_template = String::with_capacity(insert_max * 8 + 43); insert_template.push_str("INSERT INTO json_dump (packet) Values "); @@ -30,6 +30,11 @@ fn query_string(insert_max: &usize) -> String { insert_template } +fn smallest_prime_divisor(remainder: usize ) -> usize { + let smallest_divisor: usize = (2..(remainder/2)).into_par_iter().find_first(|x| remainder % x == 0).unwrap(); + smallest_divisor +} + #[tokio::main(core_threads = 4)] // By default, tokio_postgres uses the tokio crate as its runtime. async fn main() -> Result<(), Error> { /* Init values from file */ @@ -88,8 +93,15 @@ async fn main() -> Result<(), Error> { match config.is_device { false => for _pcap_file in pcap_map.keys() { + + // TODO: Tuning vector capacity according to actuarial excpectation, mean average & std dev of packet size let v: Vec = parser::parse(&_pcap_file, &config.filter); + //let mut v = Vec::::with_capacity(35536); + //v.extend(parser::parse(&_pcap_file, &config.filter)); + let packets_serialized = serializer::serialize_packets(v); + //let mut packets_serialized = Vec::::with_capacity(35536); + //packets_serialized.extend(serializer::serialize_packets(v)); /* Query */ @@ -111,7 +123,7 @@ async fn main() -> Result<(), Error> { false => { let insert_str = query_string(&config.insert_max); let statement = client.prepare(&insert_str).await?; - + for _i in 0..chunk_count { let (_input, _) = packets_serialized.split_at(config.insert_max); client @@ -123,7 +135,7 @@ async fn main() -> Result<(), Error> { println!("Chunks, total:{}", chunk_count); println!("Chunks, remainder:{}", remainder); - if remainder > 0 { + if remainder > 0 { let rem_str = query_string(&remainder); let statement_remainder = client.prepare(&rem_str).await?; let (_garbage, _input) = diff --git a/src/parser.json b/src/parser.json index 0538f38..c364cd4 100644 --- a/src/parser.json +++ b/src/parser.json @@ -3,7 +3,7 @@ "filter": "tcp && ip6", "from_device": false, "parse_device": "enp7s0", - "pcap_file": "../target/wohnung2.pcapng", + "pcap_file": "../target/arp_test.pcapng", "pcap_dir": "../target", "database_user": "postgres", "database_host": "localhost", diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 1f76be0..d19ef26 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -12,6 +12,8 @@ const ETH_P_IPV6: usize = 0xDD86; const ETH_P_IP: usize = 0x08; const TCP: usize = 0x06; const UDP: usize = 0x11; +const ETH_P_ARP: usize = 0x0608; +const ETH_P_RARP: usize = 0x3580; fn build_ether() -> packet_handler::EtherHeader { packet_handler::EtherHeader { @@ -31,6 +33,7 @@ pub struct QryData { pub ipv6_header: Option, pub tcp_header: Option, pub udp_header: Option, + pub arp_header: Option, pub reg_res: Option, } @@ -57,6 +60,7 @@ pub fn parse(parse_file: &std::path::Path, filter_str: &str) -> Vec { ipv6_header: None::, tcp_header: None::, udp_header: None::, + arp_header: None::, reg_res: None::, }; let mut v: Vec = Vec::new(); @@ -120,6 +124,9 @@ pub fn parse(parse_file: &std::path::Path, filter_str: &str) -> Vec { _ => println!("network protocol not implemented"), } } + ETH_P_ARP | ETH_P_RARP => { + me.arp_header = Some(packet_handler::arp_handler(packet.data)).unwrap(); + } _ => println!("network protocol not implemented"), } v.push(QryData { @@ -131,6 +138,7 @@ pub fn parse(parse_file: &std::path::Path, filter_str: &str) -> Vec { ipv6_header: me.ipv6_header, tcp_header: me.tcp_header, udp_header: me.udp_header, + arp_header: me.arp_header, reg_res: me.reg_res, }); } @@ -149,6 +157,7 @@ pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) -> ipv6_header: None::, tcp_header: None::, udp_header: None::, + arp_header: None::, reg_res: None::, }; let mut v: Vec = Vec::new(); @@ -212,6 +221,9 @@ pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) -> _ => println!("network protocol not implemented"), } } + ETH_P_ARP | ETH_P_RARP => { + me.arp_header = Some(packet_handler::arp_handler(packet.data)).unwrap(); + } _ => println!("network protocol not implemented"), } v.push(QryData { @@ -223,6 +235,7 @@ pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) -> ipv6_header: me.ipv6_header, tcp_header: me.tcp_header, udp_header: me.udp_header, + arp_header: me.arp_header, reg_res: me.reg_res, }); diff --git a/src/parser/packet_handler.rs b/src/parser/packet_handler.rs index baa9b6b..766255a 100644 --- a/src/parser/packet_handler.rs +++ b/src/parser/packet_handler.rs @@ -24,15 +24,13 @@ pub struct EtherHeader { pub ether_type: i32, } -// TODO: implement ethernet vlan shim header fields +// TODO: implement optional ethernet vlan shim header fields pub fn ethernet_handler(packet_data: &[u8]) -> EtherHeader { let mut _ether_dhost: [u8; ETH_ALEN] = [0; ETH_ALEN]; let mut _ether_shost: [u8; ETH_ALEN] = [0; ETH_ALEN]; let mut _ether_type: u16 = 0; _ether_dhost.clone_from_slice(&packet_data[0..ETH_ALEN]); - - //println!("{:?}", (&(_ether_dhost).to_owned())); _ether_shost.clone_from_slice(&packet_data[ETH_ALEN..ETH_ALEN * 2]); _ether_type = LittleEndian::read_u16(&packet_data[ETH_ALEN * 2..(ETH_ALEN * 2) + ETH_TLEN]); @@ -267,71 +265,110 @@ pub fn tcp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option { } /* arp */ -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ArpHeader { - pub htype: u32, - pub ptype: u32, - pub hlen: u32, - pub plen: u32, - pub oper: u32, - pub sha: String, - pub spa: IpAddr, - pub tha: String, - pub tpa: IpAddr, -} +//#[derive(Debug, Clone, Serialize, Deserialize)] +//pub struct ArpHeader { +// pub htype: u32, +// pub ptype: u32, +// pub hlen: u32, +// pub plen: u32, +// pub oper: u32, +// pub sha: String, +// pub spa: IpAddr, +// pub tha: String, +// pub tpa: IpAddr, +//} // u8, get_source_address, _: 103, 96, 4; // u32, into Ipv4Addr, get_destination_address, _: 159, 128; -bitfield! { - struct BitfieldArpHeader ( MSB0 [u8] ); - impl Debug; - u32; - get_htype, _: 0, 1; - get_ptype, _: 2, 3; - get_hlen, _: 4; - get_plen, _: 5; - get_oper, _: 6, 7; - get_sha, _: 8,13; - get_tha, _: 18, 23; - u8, get_spa, _: 17, 14, 4; - u32, into Ipv4Addr, get_tpa, _: 28, 24; -} - -impl + AsMut<[u8]>> BitfieldArpHeader { - fn get_spa_as_ip_addr(&self) -> Ipv4Addr { - let mut src = [0; 4]; - for (i, src) in src.iter_mut().enumerate() { - *src = self.get_spa(i); - } - src.into() - } -} - +//bitfield! { +// struct BitfieldArpHeader ( MSB0 [u8] ); +// impl Debug; +// u32; +// get_htype, _: 0, 1; +// get_ptype, _: 2, 3; +// get_hlen, _: 4; +// get_plen, _: 5; +// get_oper, _: 6, 7; +// get_sha, _: 8,13; +// get_tha, _: 18, 23; +// u8, get_spa, _: 17, 14, 4; +// u32, into Ipv4Addr, get_tpa, _: 28, 24; +//} +// +//impl + AsMut<[u8]>> BitfieldArpHeader { +// fn get_spa_as_ip_addr(&self) -> Ipv4Addr { +// let mut src = [0; 4]; +// for (i, src) in src.iter_mut().enumerate() { +// *src = self.get_spa(i); +// } +// src.into() +// } +//} // TODO: Fix this crap +//pub fn arp_handler(packet_data: &[u8]) -> Option { +// let mut raw_hdr: [u8; 28] = [0; 28]; +// raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN..ETHER_HDRLEN + 28]); +// +// let arp_header = BitfieldArpHeader(raw_hdr); +// let _sha: [u8; 6] = [0; 6]; +// let _tha: [u8; 6] = [0; 6]; +// raw_hdr[8..13].copy_from_slice(&_sha); +// raw_hdr[18..23].copy_from_slice(&_tha); +// +// Some(ArpHeader { +// htype: arp_header.get_htype(), +// ptype: arp_header.get_ptype(), +// hlen: arp_header.get_hlen().into(), +// plen: arp_header.get_plen().into(), +// oper: arp_header.get_oper(), +// sha: MacAddress::new(_sha as Eui48).to_hex_string(), +// //MacAddress::new(arp_header.get_sha() as Eui48).to_hex_string(), +// spa: IpAddr::V4(arp_header.get_spa_as_ip_addr()), +// tha: MacAddress::new(_tha as Eui48).to_hex_string(), +// tpa: IpAddr::V4(arp_header.get_tpa()), +// }) +//} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct ArpHeader { + pub htype: u16, + pub ptype: u16, + pub hlen: u8, + pub plen: u8, + pub oper: u16, + pub sha: Eui48, + pub spa: IpAddr, + pub tha: Eui48, + pub tpa: IpAddr, +} + pub fn arp_handler(packet_data: &[u8]) -> Option { let mut raw_hdr: [u8; 28] = [0; 28]; - raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN..ETHER_HDRLEN + 28]); + raw_hdr.copy_from_slice( + &packet_data[ETHER_HDRLEN .. ETHER_HDRLEN + 28] + ); - let arp_header = BitfieldArpHeader(raw_hdr); - let _sha: [u8; 6] = [0; 6]; - let _tha: [u8; 6] = [0; 6]; - raw_hdr[8..13].copy_from_slice(&_sha); - raw_hdr[18..23].copy_from_slice(&_tha); + let mut _sha: [u8; 6] = [0; 6]; + let mut _tha: [u8; 6] = [0; 6]; + + _sha.clone_from_slice(&raw_hdr[8..14]); + _tha.clone_from_slice(&raw_hdr[18..24]); - Some(ArpHeader { - htype: arp_header.get_htype(), - ptype: arp_header.get_ptype(), - hlen: arp_header.get_hlen().into(), - plen: arp_header.get_plen().into(), - oper: arp_header.get_oper(), - sha: MacAddress::new(_sha as Eui48).to_hex_string(), - //MacAddress::new(arp_header.get_sha() as Eui48).to_hex_string(), - spa: IpAddr::V4(arp_header.get_spa_as_ip_addr()), - tha: MacAddress::new(_tha as Eui48).to_hex_string(), - tpa: IpAddr::V4(arp_header.get_tpa()), + Some(ArpHeader{ + htype: BigEndian::read_u16(&raw_hdr[0..2]), + ptype: BigEndian::read_u16(&raw_hdr[2..4]), + hlen: raw_hdr[4], + plen: raw_hdr[5], + oper: BigEndian::read_u16(&raw_hdr[6..8]), + //sha: MacAddress::new(_sha as Eui48).to_hex_string().to_owned(), + sha: _sha, + spa: IpAddr::V4(Ipv4Addr::from(BigEndian::read_u32(&raw_hdr[14..18]))), + //tha: MacAddress::new( _tha as Eui48 ).to_hex_string().to_owned(), + tha: _tha, + tpa: IpAddr::V4(Ipv4Addr::from(BigEndian::read_u32(&raw_hdr[24..28]))), }) } - + /* udp */ #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub struct UdpHeader { diff --git a/src/serializer/mod.rs b/src/serializer/mod.rs index e93093d..39403f0 100644 --- a/src/serializer/mod.rs +++ b/src/serializer/mod.rs @@ -1,8 +1,8 @@ extern crate serde_json; +use crate::parser; use rayon::prelude::*; use serde::ser::{Serialize, SerializeStruct, Serializer}; use serde_json::json; -use crate::parser; impl Serialize for parser::QryData { fn serialize(&self, serializer: S) -> Result @@ -19,6 +19,7 @@ impl Serialize for parser::QryData { state.serialize_field("ipv6_header", &self.ipv6_header)?; state.serialize_field("tcp_header", &self.tcp_header)?; state.serialize_field("udp_header", &self.udp_header)?; + state.serialize_field("arp_header", &self.arp_header)?; state.serialize_field("data", &self.data)?; state.serialize_field("reg_res", &self.reg_res)?; state.end()