stefan
/
tpcpr
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

udp fully implemented + hefty performance boost

This commit is contained in:
gurkenhabicht 2020-05-22 19:09:05 +02:00
parent 4d3d1c98d6
commit d70a882245
5 changed files with 111 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.toml
View File

@ -6,6 +6,9 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
#[profile.dev]
#opt-level = 3
[dependencies] [dependencies]
tokio-postgres = { version="0.5.4", features = ["runtime","with-eui48-0_4","with-serde_json-1"] } tokio-postgres = { version="0.5.4", features = ["runtime","with-eui48-0_4","with-serde_json-1"] }
tokio = { version = "0.2", features = ["full"] } tokio = { version = "0.2", features = ["full"] }

7
src/main.rs
View File

@ -8,6 +8,8 @@ use std::fs::File;
mod parser; mod parser;
use tokio_postgres::types::ToSql; use tokio_postgres::types::ToSql;
use tokio_postgres::{Error, NoTls}; use tokio_postgres::{Error, NoTls};
use std::fs;
impl Serialize for parser::QryData { impl Serialize for parser::QryData {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@ -23,6 +25,7 @@ impl Serialize for parser::QryData {
state.serialize_field("ipv4_header", &self.ipv4_header)?; state.serialize_field("ipv4_header", &self.ipv4_header)?;
state.serialize_field("ipv6_header", &self.ipv6_header)?; state.serialize_field("ipv6_header", &self.ipv6_header)?;
state.serialize_field("tcp_header", &self.tcp_header)?; state.serialize_field("tcp_header", &self.tcp_header)?;
state.serialize_field("udp_header", &self.udp_header)?;
state.serialize_field("data", &self.data)?; state.serialize_field("data", &self.data)?;
state.serialize_field("reg_res", &self.reg_res)?; state.serialize_field("reg_res", &self.reg_res)?;
state.end() state.end()
@ -89,6 +92,8 @@ async fn main() -> Result<(), Error> {
eprintln!("connection error: {}", e); eprintln!("connection error: {}", e);
} }
}); });
let metadata = fs::metadata(pcap_file).unwrap();
println!("{:?}", &metadata.file_type());
client client
.execute("DROP TABLE IF EXISTS json_dump", &[]) .execute("DROP TABLE IF EXISTS json_dump", &[])
@ -135,7 +140,7 @@ async fn main() -> Result<(), Error> {
println!("Packets, total:{:?}", packets_serialized.len()); println!("Packets, total:{:?}", packets_serialized.len());
println!("Chunks, total:{}", chunk_count); println!("Chunks, total:{}", chunk_count);
println!("Chunks, remainder{}", remainder); println!("Chunks, remainder:{}", remainder);
if remainder > 0 { if remainder > 0 {
let rem_str = query_string(&remainder); let rem_str = query_string(&remainder);

2
src/parser.json
View File

@ -1,6 +1,6 @@
{ {
"insert_max": 20000, "insert_max": 20000,
"filter": "tcp && ip6", "filter": "udp && ip6",
"from_device": false, "from_device": false,
"parse_device": "enp7s0", "parse_device": "enp7s0",
"pcap_file": "../target/wohnung2.pcapng", "pcap_file": "../target/wohnung2.pcapng",

92
src/parser.rs
View File

@ -4,7 +4,6 @@ extern crate eui48;
mod packet_handler; mod packet_handler;
use eui48::MacAddress; use eui48::MacAddress;
use pcap::Capture; use pcap::Capture;
use regex::bytes::Match;
use regex::bytes::Regex; use regex::bytes::Regex;
use std::str; use std::str;
@ -12,6 +11,7 @@ use std::str;
const ETH_P_IPV6: usize = 0xDD86; const ETH_P_IPV6: usize = 0xDD86;
const ETH_P_IP: usize = 0x08; const ETH_P_IP: usize = 0x08;
const TCP: usize = 0x06; const TCP: usize = 0x06;
const UDP: usize = 0x11;
fn build_ether() -> packet_handler::EtherHeader { fn build_ether() -> packet_handler::EtherHeader {
packet_handler::EtherHeader { packet_handler::EtherHeader {
@ -30,13 +30,13 @@ pub struct QryData {
pub ipv4_header: Option<packet_handler::IpV4Header>, pub ipv4_header: Option<packet_handler::IpV4Header>,
pub ipv6_header: Option<packet_handler::IpV6Header>, pub ipv6_header: Option<packet_handler::IpV6Header>,
pub tcp_header: Option<packet_handler::TcpHeader>, pub tcp_header: Option<packet_handler::TcpHeader>,
pub udp_header: Option<packet_handler::UdpHeader>,
pub reg_res: Option<String>, pub reg_res: Option<String>,
} }
fn flag_carnage(re: &Regex, payload: &[u8]) -> Option<String> { fn flag_carnage(re: &Regex, payload: &[u8]) -> Option<String> {
let mut flags: String = String::new() ; let mut flags: String = String::new() ;
for mat in re.find_iter(payload) { for mat in re.find_iter(payload) {
// println!("{:?}", std::str::from_utf8(mat.as_bytes()).unwrap());
flags.push_str( std::str::from_utf8(mat.as_bytes()).unwrap() ); flags.push_str( std::str::from_utf8(mat.as_bytes()).unwrap() );
} }
match 0 < flags.len() { match 0 < flags.len() {
@ -56,6 +56,7 @@ pub fn parse(parse_file: &str, filter_str: &str) -> Vec<QryData> {
ipv4_header: None::<packet_handler::IpV4Header>, ipv4_header: None::<packet_handler::IpV4Header>,
ipv6_header: None::<packet_handler::IpV6Header>, ipv6_header: None::<packet_handler::IpV6Header>,
tcp_header: None::<packet_handler::TcpHeader>, tcp_header: None::<packet_handler::TcpHeader>,
udp_header: None::<packet_handler::UdpHeader>,
reg_res: None::<String>, reg_res: None::<String>,
}; };
let mut v: Vec<QryData> = Vec::new(); let mut v: Vec<QryData> = Vec::new();
@ -72,8 +73,8 @@ pub fn parse(parse_file: &str, filter_str: &str) -> Vec<QryData> {
ETH_P_IP => { ETH_P_IP => {
me.ipv6_header = None::<packet_handler::IpV6Header>; me.ipv6_header = None::<packet_handler::IpV6Header>;
me.ipv4_header = Some(packet_handler::ip_handler(packet.data)).unwrap(); me.ipv4_header = Some(packet_handler::ip_handler(packet.data)).unwrap();
if TCP == me.ipv4_header.unwrap().ip_protocol as usize { match me.ipv4_header.unwrap().ip_protocol as usize {
me.tcp_header = Some(packet_handler::tcp_handler( TCP => {me.tcp_header = Some(packet_handler::tcp_handler(
me.ipv4_header.unwrap().ip_ihl, me.ipv4_header.unwrap().ip_ihl,
packet.data, packet.data,
)) ))
@ -83,22 +84,44 @@ pub fn parse(parse_file: &str, filter_str: &str) -> Vec<QryData> {
me.tcp_header.unwrap().data_offset, me.tcp_header.unwrap().data_offset,
packet.data, packet.data,
); );
},
UDP => { me.udp_header = Some(packet_handler::udp_handler(
me.ipv4_header.unwrap().ip_ihl,
packet.data
)).unwrap();
me.data = packet_handler::payload_handler(
me.ipv4_header.unwrap().ip_ihl,
7,
packet.data );
},
_ => println!("network protocol not implemented"),
} }
} },
ETH_P_IPV6 => { ETH_P_IPV6 => {
me.ipv4_header = None::<packet_handler::IpV4Header>; me.ipv4_header = None::<packet_handler::IpV4Header>;
me.ipv6_header = Some(packet_handler::ipv6_handler(packet.data)).unwrap(); me.ipv6_header = Some(packet_handler::ipv6_handler(packet.data)).unwrap();
if TCP == me.ipv6_header.unwrap().next_header as usize { match me.ipv6_header.unwrap().next_header as usize {
me.tcp_header = Some(packet_handler::tcp_handler(10, packet.data)).unwrap(); TCP => {
me.data = packet_handler::payload_handler( me.tcp_header = Some(packet_handler::tcp_handler(10, packet.data)).unwrap();
me.data = packet_handler::payload_handler(
10, 10,
me.tcp_header.unwrap().data_offset, me.tcp_header.unwrap().data_offset,
packet.data, packet.data,
); );
},
UDP => { me.udp_header = Some(packet_handler::udp_handler(
10,
packet.data
)).unwrap();
me.data = packet_handler::payload_handler(
10,
7,
packet.data);
},
_ => println!("network protocol not implemented"),
} }
} },
_ => println!("network protocol not implemented"),
_ => println!("network protocol not implemented"),
} }
v.push(QryData { v.push(QryData {
id: 0, id: 0,
@ -108,6 +131,7 @@ pub fn parse(parse_file: &str, filter_str: &str) -> Vec<QryData> {
ipv4_header: me.ipv4_header, ipv4_header: me.ipv4_header,
ipv6_header: me.ipv6_header, ipv6_header: me.ipv6_header,
tcp_header: me.tcp_header, tcp_header: me.tcp_header,
udp_header: me.udp_header,
reg_res: me.reg_res, reg_res: me.reg_res,
}); });
} }
@ -125,6 +149,7 @@ pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) ->
ipv4_header: None::<packet_handler::IpV4Header>, ipv4_header: None::<packet_handler::IpV4Header>,
ipv6_header: None::<packet_handler::IpV6Header>, ipv6_header: None::<packet_handler::IpV6Header>,
tcp_header: None::<packet_handler::TcpHeader>, tcp_header: None::<packet_handler::TcpHeader>,
udp_header: None::<packet_handler::UdpHeader>,
reg_res: None::<String>, reg_res: None::<String>,
}; };
let mut v: Vec<QryData> = Vec::new(); let mut v: Vec<QryData> = Vec::new();
@ -141,8 +166,8 @@ pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) ->
ETH_P_IP => { ETH_P_IP => {
me.ipv6_header = None::<packet_handler::IpV6Header>; me.ipv6_header = None::<packet_handler::IpV6Header>;
me.ipv4_header = Some(packet_handler::ip_handler(packet.data)).unwrap(); me.ipv4_header = Some(packet_handler::ip_handler(packet.data)).unwrap();
if TCP == me.ipv4_header.unwrap().ip_protocol as usize { match me.ipv4_header.unwrap().ip_protocol as usize {
me.tcp_header = Some(packet_handler::tcp_handler( TCP => {me.tcp_header = Some(packet_handler::tcp_handler(
me.ipv4_header.unwrap().ip_ihl, me.ipv4_header.unwrap().ip_ihl,
packet.data, packet.data,
)) ))
@ -152,22 +177,44 @@ pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) ->
me.tcp_header.unwrap().data_offset, me.tcp_header.unwrap().data_offset,
packet.data, packet.data,
); );
},
UDP => { me.udp_header = Some(packet_handler::udp_handler(
me.ipv4_header.unwrap().ip_ihl,
packet.data
)).unwrap();
me.data = packet_handler::payload_handler(
me.ipv4_header.unwrap().ip_ihl,
7,
packet.data );
},
_ => println!("network protocol not implemented"),
} }
} },
ETH_P_IPV6 => { ETH_P_IPV6 => {
me.ipv4_header = None::<packet_handler::IpV4Header>; me.ipv4_header = None::<packet_handler::IpV4Header>;
me.ipv6_header = Some(packet_handler::ipv6_handler(packet.data)).unwrap(); me.ipv6_header = Some(packet_handler::ipv6_handler(packet.data)).unwrap();
if TCP == me.ipv6_header.unwrap().next_header as usize { match me.ipv6_header.unwrap().next_header as usize {
me.tcp_header = Some(packet_handler::tcp_handler(10, packet.data)).unwrap(); TCP => {
me.data = packet_handler::payload_handler( me.tcp_header = Some(packet_handler::tcp_handler(10, packet.data)).unwrap();
me.data = packet_handler::payload_handler(
10, 10,
me.tcp_header.unwrap().data_offset, me.tcp_header.unwrap().data_offset,
packet.data, packet.data,
); );
},
UDP => { me.udp_header = Some(packet_handler::udp_handler(
10,
packet.data
)).unwrap();
me.data = packet_handler::payload_handler(
10,
7,
packet.data);
},
_ => println!("network protocol not implemented"),
} }
} },
_ => println!("network protocol not implemented"),
_ => println!("network protocol not implemented"),
} }
v.push(QryData { v.push(QryData {
id: 0, id: 0,
@ -177,9 +224,10 @@ pub fn parse_device(parse_device: &str, filter_str: &str, insert_max: &usize) ->
ipv4_header: me.ipv4_header, ipv4_header: me.ipv4_header,
ipv6_header: me.ipv6_header, ipv6_header: me.ipv6_header,
tcp_header: me.tcp_header, tcp_header: me.tcp_header,
reg_res: me.reg_res, udp_header: me.udp_header,
reg_res: me.reg_res,
}); });
if &v.len() >= insert_max { if &v.len() >= insert_max {
break 'parse; break 'parse;
} }

70
src/parser/packet_handler.rs
View File

@ -8,6 +8,8 @@ use eui48::{Eui48, MacAddress};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
// TODO Benchmark slice refrence vs. bitfield
/* ethernet */ /* ethernet */
const ETH_ALEN: usize = 6; const ETH_ALEN: usize = 6;
const ETH_TLEN: usize = 2; const ETH_TLEN: usize = 2;
@ -15,13 +17,14 @@ const ETHER_HDRLEN: usize = 14;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct EtherHeader { pub struct EtherHeader {
// pub ether_dhost: MacAddress, // pub ether_dhost: MacAddress, // ommitted because serde_json serializer doen 't recognize MacAddress type
// pub ether_shost: MacAddress, // pub ether_shost: MacAddress,
pub ether_dhost: String, pub ether_dhost: String,
pub ether_shost: String, pub ether_shost: String,
pub ether_type: i32, pub ether_type: i32,
} }
// TODO: implement ethernet vlan shim header fields
pub fn ethernet_handler(packet_data: &[u8]) -> EtherHeader { pub fn ethernet_handler(packet_data: &[u8]) -> EtherHeader {
let mut _ether_dhost: [u8; ETH_ALEN] = [0; ETH_ALEN]; let mut _ether_dhost: [u8; ETH_ALEN] = [0; ETH_ALEN];
let mut _ether_shost: [u8; ETH_ALEN] = [0; ETH_ALEN]; let mut _ether_shost: [u8; ETH_ALEN] = [0; ETH_ALEN];
@ -90,12 +93,9 @@ impl<T: AsRef<[u8]> + AsMut<[u8]>> BitfieldIpV4Header<T> {
} }
pub fn ip_handler(packet_data: &[u8]) -> Option<IpV4Header> { pub fn ip_handler(packet_data: &[u8]) -> Option<IpV4Header> {
let (_head, tail) = packet_data.split_at(ETHER_HDRLEN); let mut raw_hdr: [u8;20] = [0;20];
let (raw_hdr, _) = tail.split_at(20); raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN..ETHER_HDRLEN+20]);
let mut _tail: [u8; 20] = [0; 20]; let ip_header = BitfieldIpV4Header(raw_hdr);
_tail.copy_from_slice(raw_hdr);
let ip_header = BitfieldIpV4Header(_tail);
Some(IpV4Header { Some(IpV4Header {
ip_version: ip_header.get_version(), ip_version: ip_header.get_version(),
@ -129,21 +129,18 @@ pub struct IpV6Header {
} }
pub fn ipv6_handler(packet_data: &[u8]) -> Option<IpV6Header> { pub fn ipv6_handler(packet_data: &[u8]) -> Option<IpV6Header> {
let (_head, tail) = packet_data.split_at(ETHER_HDRLEN); let mut raw_hdr: [u8;40] = [0;40];
let (raw_hdr, _) = tail.split_at(40); raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN..ETHER_HDRLEN+40]);
let mut _tail: [u8; 40] = [0; 40];
_tail.copy_from_slice(raw_hdr);
//let mut rdr = Cursor::new(_tail);
Some(IpV6Header { Some(IpV6Header {
version: (&raw_hdr[0] & 0xf0) >> 4, version: (&raw_hdr[0] & 0xf0) >> 4,
traffic_class: ((&raw_hdr[0] & 0x0f) >> 4) | (&raw_hdr[1] & 0xf0 << 4), traffic_class: ((&raw_hdr[0] & 0x0f) >> 4) | (&raw_hdr[1] & 0xf0 << 4),
flow_label: BigEndian::read_u32(&[0x00, (&_tail[1] & 0x0f), _tail[2], _tail[3]]), flow_label: BigEndian::read_u32(&[0x00, (&raw_hdr[1] & 0x0f), raw_hdr[2], raw_hdr[3]]),
payload_length: BigEndian::read_u16(&[_tail[4], _tail[5]]), payload_length: BigEndian::read_u16(&[raw_hdr[4], raw_hdr[5]]),
next_header: _tail[6], next_header: raw_hdr[6],
hop_limit: _tail[7], hop_limit: raw_hdr[7],
source_address: IpAddr::V6(Ipv6Addr::from(BigEndian::read_u128(&_tail[8..24]))), // Results in correct addresses, but interval range is not default [incl..excl]. Watch out! source_address: IpAddr::V6(Ipv6Addr::from(BigEndian::read_u128(&raw_hdr[8..24]))), // Results in correct addresses, but interval range is not default [incl..excl]. Watch out!
destination_address: IpAddr::V6(Ipv6Addr::from(BigEndian::read_u128(&_tail[24..40]))), // Must be byteorder crate !? destination_address: IpAddr::V6(Ipv6Addr::from(BigEndian::read_u128(&raw_hdr[24..40]))), // Must be byteorder crate !?
}) })
} }
@ -241,11 +238,9 @@ bitfield! {
} }
pub fn tcp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<TcpHeader> { pub fn tcp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<TcpHeader> {
let (_head, tail) = packet_data.split_at(ETHER_HDRLEN + ip_hlen as usize * 4); let mut raw_hdr: [u8; 20] = [0;20];
let (raw_hdr, _) = tail.split_at(20); raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN+ip_hlen as usize * 4..ETHER_HDRLEN+ip_hlen as usize * 4 + 20]);
let mut _tail: [u8; 20] = [0; 20]; let tcp_header = BitfieldTcpHeader(raw_hdr);
_tail.copy_from_slice(raw_hdr);
let tcp_header = BitfieldTcpHeader(_tail);
Some(TcpHeader { Some(TcpHeader {
source_port: tcp_header.get_source_port(), source_port: tcp_header.get_source_port(),
@ -310,17 +305,16 @@ impl<T: AsRef<[u8]> + AsMut<[u8]>> BitfieldArpHeader<T> {
} }
} }
// TODO: Fix this crap
pub fn arp_handler(packet_data: &[u8]) -> Option<ArpHeader> { pub fn arp_handler(packet_data: &[u8]) -> Option<ArpHeader> {
let (_head, tail) = packet_data.split_at(ETHER_HDRLEN); let mut raw_hdr: [u8;28] = [0;28];
let (raw_hdr, _) = tail.split_at(28); raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN..ETHER_HDRLEN + 28]);
let mut _tail: [u8; 28] = [0; 28];
_tail.copy_from_slice(raw_hdr); let arp_header = BitfieldArpHeader(raw_hdr);
let arp_header = BitfieldArpHeader(_tail);
let _sha: [u8; 6] = [0; 6]; let _sha: [u8; 6] = [0; 6];
let _tha: [u8; 6] = [0; 6]; let _tha: [u8; 6] = [0; 6];
_tail[8..13].copy_from_slice(&_sha); raw_hdr[8..13].copy_from_slice(&_sha);
_tail[18..23].copy_from_slice(&_tha); raw_hdr[18..23].copy_from_slice(&_tha);
Some(ArpHeader { Some(ArpHeader {
htype: arp_header.get_htype(), htype: arp_header.get_htype(),
@ -346,16 +340,14 @@ pub struct UdpHeader {
} }
pub fn udp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<UdpHeader> { pub fn udp_handler(ip_hlen: u32, packet_data: &[u8]) -> Option<UdpHeader> {
let (_head, tail) = packet_data.split_at(ETHER_HDRLEN + ip_hlen as usize * 4); let mut raw_hdr: [u8;8] = [0;8];
let (raw_hdr, _) = tail.split_at(8); raw_hdr.copy_from_slice(&packet_data[ETHER_HDRLEN + ip_hlen as usize * 4..ETHER_HDRLEN + ip_hlen as usize * 4 + 8]);
let mut _tail: [u8; 8] = [0; 8];
_tail.copy_from_slice(raw_hdr);
Some(UdpHeader { Some(UdpHeader {
source_port: BigEndian::read_u16(&_tail[0..2]), source_port: BigEndian::read_u16(&raw_hdr[0..2]),
destination_port: BigEndian::read_u16(&_tail[2..4]), destination_port: BigEndian::read_u16(&raw_hdr[2..4]),
length: BigEndian::read_u16(&_tail[4..6]), length: BigEndian::read_u16(&raw_hdr[4..6]),
checksum: BigEndian::read_u16(&_tail[6..8]), checksum: BigEndian::read_u16(&raw_hdr[6..8]),
}) })
} }