|
|
@@ -0,0 +1,367 @@
|
|
|
+
|
|
|
+
|
|
|
+use std::collections::HashMap;
|
|
|
+use chrono::{Utc, Duration, Local, DateTime};
|
|
|
+use select::document::Document;
|
|
|
+use select::predicate::Name;
|
|
|
+use parse_duration::parse;
|
|
|
+use reqwest;
|
|
|
+
|
|
|
+
|
|
|
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
|
|
|
+pub struct Auction {
|
|
|
+ pub price: f32,
|
|
|
+ pub desc: String,
|
|
|
+ pub gcal: String,
|
|
|
+ pub thumb: String,
|
|
|
+ pub remaining: i64,
|
|
|
+ pub url: String,
|
|
|
+ pub timestamp: i64,
|
|
|
+ pub is_price_final: bool
|
|
|
+}
|
|
|
+
|
|
|
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
|
|
|
+pub struct Account {
|
|
|
+ // SEARCH URL: {AUCTION_URL: AUCTION}
|
|
|
+ pub queries: HashMap<String, HashMap<String, Auction>>
|
|
|
+}
|
|
|
+
|
|
|
+impl Account {
|
|
|
+ // pub fn run_queries(&mut self) {
|
|
|
+ // // self.queries = self.queries.clone().into_iter().map(|mut q| q.run()).collect();
|
|
|
+ // for url in &self.queries {
|
|
|
+ // dbg!(&url);
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ pub fn get_auctions(&self, account_name: &str) -> HashMap<String, Auction> {
|
|
|
+ match self.queries.get(account_name) {
|
|
|
+ Some(auctions) => auctions.clone(),
|
|
|
+ None => HashMap::new()
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ pub fn run_queries(&self) -> HashMap<String, HashMap<String, Auction>> {
|
|
|
+ self.queries
|
|
|
+ .clone()
|
|
|
+ .into_iter()
|
|
|
+ .map(|(url, mut auction_map)| (url.clone(), {
|
|
|
+ auction_map.extend(auctions_from_url(&url));
|
|
|
+ auction_map
|
|
|
+ }))
|
|
|
+ .map(|(url, mut auction_map)| {
|
|
|
+
|
|
|
+ (url, auction_map.into_iter().map(|(u, mut a)| {
|
|
|
+ if Utc::now().with_timezone(&Local).timestamp() > a.timestamp {
|
|
|
+ // auction has expired, mark price as final
|
|
|
+ a.is_price_final = true;
|
|
|
+ }
|
|
|
+ (u,a)
|
|
|
+ }).collect())
|
|
|
+ })
|
|
|
+ .collect()
|
|
|
+ }
|
|
|
+
|
|
|
+ pub fn updated(&self) -> Account {
|
|
|
+ Account {queries: self.run_queries()}
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+// #[derive(Serialize, Deserialize, Debug, Clone)]
|
|
|
+// pub struct Query {
|
|
|
+// pub url: String,
|
|
|
+// pub auctions: Vec<Auction>,
|
|
|
+// pub frequency: i64,
|
|
|
+// }
|
|
|
+
|
|
|
+// impl Query {
|
|
|
+// pub fn run(&mut self) -> Query {
|
|
|
+// self.auctions = parse_url(&self.url);
|
|
|
+// self.clone()
|
|
|
+// }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+// fn detect_frequency(&self) {
|
|
|
+// let _a = &self.auctions
|
|
|
+// .iter()
|
|
|
+// .map(|x| x)
|
|
|
+// .collect::<Vec<_>>();
|
|
|
+// }
|
|
|
+// }
|
|
|
+
|
|
|
+// impl Default for Query {
|
|
|
+// fn default () -> Query {
|
|
|
+// Query{
|
|
|
+// frequency: Duration::minutes(5).num_seconds(),
|
|
|
+// url: "".to_string(),
|
|
|
+// auctions: vec![]
|
|
|
+// }
|
|
|
+// }
|
|
|
+// }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+fn date_to_gcal(date: DateTime<Local>) -> String {
|
|
|
+ format!("{}",
|
|
|
+ date.format("%Y%m%dT%H%M00/%Y%m%dT%H%M00")
|
|
|
+ )
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+/// Parse a relative offset "x Tage HH:MM" into proper time
|
|
|
+fn parse_remaining(timestring: &String) -> Option<Duration> {
|
|
|
+ match parse(&format!("{} minutes", timestring.replace(":", " hours ").replace("Tage", "days"))).ok() {
|
|
|
+ Some(old_duration) => Duration::from_std(old_duration).ok(),
|
|
|
+ None => None
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+fn parse_price(price: &str) -> Option<f32> {
|
|
|
+ price.to_string()
|
|
|
+ .replace(".", "")
|
|
|
+ .replace(",", ".")
|
|
|
+ .replace(" EUR", "")
|
|
|
+ .parse().ok()
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+fn parse_url(url: &str) -> Vec<Auction> {
|
|
|
+ let mut auctions = vec![];
|
|
|
+ // dbg!(&url);
|
|
|
+
|
|
|
+ if let Ok(mut resp) = reqwest::get(url) {
|
|
|
+
|
|
|
+ if !resp.status().is_success() {
|
|
|
+ println!("ERR {:?}", resp.text());
|
|
|
+ return auctions;
|
|
|
+ }
|
|
|
+
|
|
|
+ let text = resp.text().unwrap_or("".to_string());
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ for node in Document::from_read(text.as_bytes())
|
|
|
+ .unwrap()
|
|
|
+ // .find(Name("a"))
|
|
|
+ // .filter(|n| n.attr("href").is_some())
|
|
|
+ // .filter(|n| n.attr("href").unwrap().contains("item.php?id="))
|
|
|
+ .find(Name("tr"))
|
|
|
+ .filter(|x| x.attr("bgcolor").is_some())
|
|
|
+ .filter(|x| x.attr("align").is_none())
|
|
|
+
|
|
|
+ {
|
|
|
+
|
|
|
+ // Get auction name
|
|
|
+ if let Some(name) = node
|
|
|
+ .children().into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
+ .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| x.name() == Some("a"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .map(|x| x.text())
|
|
|
+ // .filter(|x| x.name() == Some("Text"))
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+
|
|
|
+ // If we have the name, go on finding other details
|
|
|
+ // instantiate mutable Auction
|
|
|
+ let mut auction = Auction::default();
|
|
|
+ auction.desc = name.clone();
|
|
|
+ auction.thumb = format!("http://egun.de/market/images/picture.gif");
|
|
|
+
|
|
|
+
|
|
|
+ // get image
|
|
|
+ if let Some(img) = node
|
|
|
+ .children().into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
+ .filter(|x| x.attr("align") == Some("center"))
|
|
|
+ .flat_map(|x| x.descendants())
|
|
|
+ .filter(|x| x.name() == Some("img"))
|
|
|
+ .filter(|x| match x.attr("src") {
|
|
|
+ Some(src) => src.contains("cache"),
|
|
|
+ None => false
|
|
|
+ } )
|
|
|
+ .map(|x| x.attr("src").unwrap()) // we just tested
|
|
|
+
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+ auction.thumb = format!("http://egun.de/market/{}", img);
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // get price
|
|
|
+ if let Some(price) = node
|
|
|
+ .children()
|
|
|
+ .filter(|x| x.text().contains("EUR"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .map(|x| parse_price(&x.text()))
|
|
|
+ .flat_map(|x| x)
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+ auction.price = price.clone();
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // get article url
|
|
|
+ if let Some(url) = node
|
|
|
+ .children().into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
+ .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| x.name() == Some("a"))
|
|
|
+ .map(|x| x.attr("href"))
|
|
|
+ .filter_map(|x| x)
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+ auction.url = format!("http://egun.de/market/{}", url);
|
|
|
+ }
|
|
|
+
|
|
|
+ // TODO: check if https://doc.rust-lang.org/std/time/struct.SystemTime.html works too
|
|
|
+ if let Some(remaining) = parse_remaining(
|
|
|
+ &node
|
|
|
+ .children()
|
|
|
+ .filter(|x| x.attr("align") == Some("center"))
|
|
|
+ .filter(|x| x.attr("nowrap").is_some())
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| !x.text().is_empty())
|
|
|
+ .map(|x| x.text())
|
|
|
+ .collect::<Vec<_>>()
|
|
|
+ .join(" ")
|
|
|
+ ){
|
|
|
+ // dbg!(&t_remaining.children());
|
|
|
+
|
|
|
+ let end_date = Utc::now().with_timezone(&Local) + remaining;
|
|
|
+ auction.gcal = format!("http://www.google.com/calendar/event?action=TEMPLATE&dates={}&text={}&location=&details=", date_to_gcal(end_date), auction.desc);
|
|
|
+ auction.remaining = remaining.num_seconds();
|
|
|
+ auction.timestamp = end_date.timestamp();
|
|
|
+ // println!("ENDS\t{:?}", date_to_gcal(remaining));
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ auctions.push(auction);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ auctions
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+fn auctions_from_url(url: &str) -> HashMap<String, Auction> {
|
|
|
+ let mut auctions = HashMap::new();
|
|
|
+ // dbg!(&url);
|
|
|
+
|
|
|
+ if let Ok(mut resp) = reqwest::get(url) {
|
|
|
+
|
|
|
+ if !resp.status().is_success() {
|
|
|
+ println!("ERR {:?}", resp.text());
|
|
|
+ return auctions;
|
|
|
+ }
|
|
|
+
|
|
|
+ let text = resp.text().unwrap_or("".to_string());
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ for node in Document::from_read(text.as_bytes())
|
|
|
+ .unwrap()
|
|
|
+ // .find(Name("a"))
|
|
|
+ // .filter(|n| n.attr("href").is_some())
|
|
|
+ // .filter(|n| n.attr("href").unwrap().contains("item.php?id="))
|
|
|
+ .find(Name("tr"))
|
|
|
+ .filter(|x| x.attr("bgcolor").is_some())
|
|
|
+ .filter(|x| x.attr("align").is_none())
|
|
|
+
|
|
|
+ {
|
|
|
+
|
|
|
+ // Get auction name
|
|
|
+ if let Some(name) = node
|
|
|
+ .children().into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
+ .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| x.name() == Some("a"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .map(|x| x.text())
|
|
|
+ // .filter(|x| x.name() == Some("Text"))
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+
|
|
|
+ // If we have the name, go on finding other details
|
|
|
+ // instantiate mutable Auction
|
|
|
+ let mut auction = Auction::default();
|
|
|
+ auction.desc = name.clone();
|
|
|
+ auction.thumb = format!("http://egun.de/market/images/picture.gif");
|
|
|
+
|
|
|
+
|
|
|
+ // get image
|
|
|
+ if let Some(img) = node
|
|
|
+ .children().into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
+ .filter(|x| x.attr("align") == Some("center"))
|
|
|
+ .flat_map(|x| x.descendants())
|
|
|
+ .filter(|x| x.name() == Some("img"))
|
|
|
+ .filter(|x| match x.attr("src") {
|
|
|
+ Some(src) => src.contains("cache"),
|
|
|
+ None => false
|
|
|
+ } )
|
|
|
+ .map(|x| x.attr("src").unwrap()) // we just tested
|
|
|
+
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+ auction.thumb = format!("http://egun.de/market/{}", img);
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // get price
|
|
|
+ if let Some(price) = node
|
|
|
+ .children()
|
|
|
+ .filter(|x| x.text().contains("EUR"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .map(|x| parse_price(&x.text()))
|
|
|
+ .flat_map(|x| x)
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+ auction.price = price.clone();
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // get article url
|
|
|
+ if let Some(url) = node
|
|
|
+ .children().into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
+ .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| x.name() == Some("a"))
|
|
|
+ .map(|x| x.attr("href"))
|
|
|
+ .filter_map(|x| x)
|
|
|
+ .collect::<Vec<_>>().get(0) {
|
|
|
+ auction.url = format!("http://egun.de/market/{}", url);
|
|
|
+ }
|
|
|
+
|
|
|
+ // TODO: check if https://doc.rust-lang.org/std/time/struct.SystemTime.html works too
|
|
|
+ if let Some(remaining) = parse_remaining(
|
|
|
+ &node
|
|
|
+ .children()
|
|
|
+ .filter(|x| x.attr("align") == Some("center"))
|
|
|
+ .filter(|x| x.attr("nowrap").is_some())
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| !x.text().is_empty())
|
|
|
+ .map(|x| x.text())
|
|
|
+ .collect::<Vec<_>>()
|
|
|
+ .join(" ")
|
|
|
+ ){
|
|
|
+ // dbg!(&t_remaining.children());
|
|
|
+
|
|
|
+ let end_date = Utc::now().with_timezone(&Local) + remaining;
|
|
|
+ auction.gcal = format!("http://www.google.com/calendar/event?action=TEMPLATE&dates={}&text={}&location=&details=", date_to_gcal(end_date), auction.desc);
|
|
|
+ auction.remaining = remaining.num_seconds();
|
|
|
+ auction.timestamp = end_date.timestamp();
|
|
|
+ // println!("ENDS\t{:?}", date_to_gcal(remaining));
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ auctions.insert(auction.url.clone(), auction);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ auctions
|
|
|
+}
|
