#[macro_use] extern crate serde_derive; use core::cmp::Ordering::Equal; use reqwest; use chrono::{Utc, Duration, Local, DateTime}; use select::document::Document; use select::predicate::Name; use parse_duration::parse; use serde_json; use std::io::BufWriter; use std::io::BufReader; use std::fs::File; use std::{thread, time}; use std::collections::HashMap; /// Fucking egun is a mess. It does not even use css and is built using tables. This is an attempt to parse it. #[derive(Serialize, Deserialize, Debug, Default, Clone)] struct Auction { price: f32, desc: String, gcal: String, thumb: String, remaining: i64, url: String, timestamp: i64, is_price_final: bool } #[derive(Serialize, Deserialize, Debug, Clone)] struct Query { url: String, auctions: Vec, frequency: i64, } impl Query { fn run(&mut self) { self.auctions = parse_url(&self.url); } fn detect_frequency(&self) { let _a = &self.auctions .iter() .map(|x| x) .collect::>(); } } impl Default for Query { fn default () -> Query { Query{ frequency: Duration::minutes(5).num_seconds(), url: "".to_string(), auctions: vec![] } } } #[derive(Serialize, Deserialize, Debug, Default)] struct Account { queries: Vec, name: String, } fn date_to_gcal(date: DateTime) -> String { format!("{}", date.format("%Y%m%dT%H%M00/%Y%m%dT%H%M00") ) } /// Parse a relative offset "x Tage HH:MM" into proper time // fn parse_end_date(timestring: &String) -> Option> { // match parse(&format!("{} minutes", timestring.replace(":", " hours ").replace("Tage", "days"))).ok() { // Some(old_duration) => match Duration::from_std(old_duration).ok() { // Some(chronoduration) => Some(Utc::now().with_timezone(&Local) + chronoduration), // None => None // } // None => None // } // } /// Parse a relative offset "x Tage HH:MM" into proper time fn parse_remaining(timestring: &String) -> Option { match parse(&format!("{} minutes", timestring.replace(":", " hours ").replace("Tage", "days"))).ok() { Some(old_duration) => Duration::from_std(old_duration).ok(), None => None } } fn parse_price(price: &str) -> Option { price.to_string() .replace(".", "") .replace(",", ".") .replace(" EUR", "") .parse().ok() } fn parse_url(url: &str) -> Vec { let mut auctions = vec![]; if let Ok(mut resp) = reqwest::get(url) { if !resp.status().is_success() { println!("ERR {:?}", resp.text()); return auctions; } let text = resp.text().unwrap_or("".to_string()); for node in Document::from_read(text.as_bytes()) .unwrap() // .find(Name("a")) // .filter(|n| n.attr("href").is_some()) // .filter(|n| n.attr("href").unwrap().contains("item.php?id=")) .find(Name("tr")) .filter(|x| x.attr("bgcolor").is_some()) .filter(|x| x.attr("align").is_none()) { // Get auction name if let Some(name) = node .children().into_iter() .filter(|x| x.name() == Some("td")) .filter(|x| x.attr("align") == Some("LEFT")) .flat_map(|x| x.children()) .filter(|x| x.name() == Some("a")) .flat_map(|x| x.children()) .map(|x| x.text()) // .filter(|x| x.name() == Some("Text")) .collect::>().get(0) { // If we have the name, go on finding other details // instantiate mutable Auction let mut auction = Auction::default(); auction.desc = name.clone(); auction.thumb = format!("http://egun.de/market/images/picture.gif"); // get image if let Some(img) = node .children().into_iter() .filter(|x| x.name() == Some("td")) .filter(|x| x.attr("align") == Some("center")) .flat_map(|x| x.descendants()) .filter(|x| x.name() == Some("img")) .filter(|x| match x.attr("src") { Some(src) => src.contains("cache"), None => false } ) .map(|x| x.attr("src").unwrap()) // we just tested .collect::>().get(0) { auction.thumb = format!("http://egun.de/market/{}", img); } // get price if let Some(price) = node .children() .filter(|x| x.text().contains("EUR")) .flat_map(|x| x.children()) .map(|x| parse_price(&x.text())) .flat_map(|x| x) .collect::>().get(0) { auction.price = price.clone(); } // get article url if let Some(url) = node .children().into_iter() .filter(|x| x.name() == Some("td")) .filter(|x| x.attr("align") == Some("LEFT")) .flat_map(|x| x.children()) .filter(|x| x.name() == Some("a")) .map(|x| x.attr("href")) .filter_map(|x| x) .collect::>().get(0) { auction.url = format!("http://egun.de/market/{}", url); } // TODO: check if https://doc.rust-lang.org/std/time/struct.SystemTime.html works too if let Some(remaining) = parse_remaining( &node .children() .filter(|x| x.attr("align") == Some("center")) .filter(|x| x.attr("nowrap").is_some()) .flat_map(|x| x.children()) .filter(|x| !x.text().is_empty()) .map(|x| x.text()) .collect::>() .join(" ") ){ // dbg!(&t_remaining.children()); let end_date = Utc::now().with_timezone(&Local) + remaining; auction.gcal = format!("http://www.google.com/calendar/event?action=TEMPLATE&dates={}&text={}&location=&details=", date_to_gcal(end_date), auction.desc); auction.remaining = remaining.num_seconds(); auction.timestamp = end_date.timestamp(); // println!("ENDS\t{:?}", date_to_gcal(remaining)); } auctions.push(auction); } } } auctions } fn daemon(queries: Vec, loaded_auctions: Vec) { // dbg!(&queries); // let _s = loaded_auctions.into_iter().map(|x| (x.url.clone(), x.clone())).collect::>(); // for auction in loaded_auctions { // if Utc::now().with_timezone(&Local).timestamp() > auction.timestamp { // //auction has expired, mark price as final // auction.is_price_final = true; // // let a = auction_map.get_mut(_key); // dbg!(&auction); // } // } let auction_map: HashMap = loaded_auctions .into_iter() .map(|mut x| { if Utc::now().with_timezone(&Local).timestamp() > x.timestamp { // auction has expired, mark price as final x.is_price_final = true; } x } ) .map(|x| (x.url.clone(), x.clone())) .collect(); // return; println!("Starting daemon with {} active queries", queries.len()); loop { // TODO: see if we can get rid of clone let mut auction_map = auction_map.clone(); // let mut auctions = vec![]; for mut query in queries.clone() { query.run(); // dbg!(&query.auctions.into_iter().map(|x| x.desc).collect::>()); // auctions.extend(query.auctions); for auction in query.auctions { dbg!(&auction.desc); let a = auction.clone(); auction_map.insert(auction.url, a); } } println!("{} auctions found", &auction_map.len()); let writer = BufWriter::new(File::create("db.json").unwrap()); let mut auctions: Vec = auction_map.into_iter().map(|(_x, y)| y).collect(); auctions.sort_by_key(|k| k.remaining); serde_json::to_writer_pretty(writer, &auctions).unwrap(); let pause = time::Duration::from_secs(300); thread::sleep(pause); } } fn main() { let reader = BufReader::new(File::open("urls.json").unwrap()); // let reader = BufReader::new(File::open("url_local.json").unwrap()); let urls: Vec = serde_json::from_reader(reader).unwrap_or(vec![]); let reader = BufReader::new(File::open("db.json").unwrap()); let auctions: Vec = serde_json::from_reader(reader).unwrap_or(vec![]); daemon( urls.iter() .map(|x| Query {url: x.to_string(), ..Default::default()}) .collect::>(), auctions ); }