|
|
@@ -1,12 +1,9 @@
|
|
|
-
|
|
|
-
|
|
|
-use std::collections::HashMap;
|
|
|
-use chrono::{Utc, Duration, Local, DateTime};
|
|
|
-use select::document::Document;
|
|
|
-use select::predicate::Name;
|
|
|
+use chrono::{DateTime, Duration, Local, Utc};
|
|
|
use parse_duration::parse;
|
|
|
use reqwest;
|
|
|
-
|
|
|
+use select::document::Document;
|
|
|
+use select::predicate::Name;
|
|
|
+use std::collections::HashMap;
|
|
|
|
|
|
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
|
|
|
pub struct Auction {
|
|
|
@@ -17,374 +14,217 @@ pub struct Auction {
|
|
|
pub remaining: i64,
|
|
|
pub url: String,
|
|
|
pub timestamp: i64,
|
|
|
- pub is_price_final: bool
|
|
|
+ pub is_price_final: bool,
|
|
|
+}
|
|
|
+
|
|
|
+impl Auction {
|
|
|
+ fn update(&mut self) {
|
|
|
+ // println!("now: {:?} auction: {:?}", Utc::now().with_timezone(&Local).timestamp(), self.timestamp);
|
|
|
+ self.is_price_final = Utc::now().with_timezone(&Local).timestamp() > self.timestamp
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
|
|
|
pub struct Query {
|
|
|
#[serde(default)]
|
|
|
pub auctions: HashMap<String, Auction>,
|
|
|
- #[serde(default)]
|
|
|
- avg_price: f32
|
|
|
+ #[serde(skip_deserializing)]
|
|
|
+ avg_price: f32,
|
|
|
}
|
|
|
|
|
|
impl Query {
|
|
|
fn get_avg_price(&mut self) -> f32 {
|
|
|
self.auctions.iter().map(|(_u, a)| a.price).sum::<f32>() / self.auctions.len() as f32
|
|
|
}
|
|
|
-}
|
|
|
|
|
|
+ fn detect_expired(&self) -> HashMap<String, Auction> {
|
|
|
+ self.auctions
|
|
|
+ .iter()
|
|
|
+ .map(|(u, mut a)| {
|
|
|
+ let mut a1 = a.clone();
|
|
|
+ a1.update();
|
|
|
+ (u.clone(), a1)
|
|
|
+ })
|
|
|
+ .collect()
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
|
|
|
pub struct Account {
|
|
|
- // SEARCH URL: {AUCTION_URL: AUCTION}
|
|
|
+ // SEARCH URL: {AUCTION_URL: AUCTION}
|
|
|
// pub queries: HashMap<String, HashMap<String, Auction>>,
|
|
|
- pub queries: HashMap<String, Query>
|
|
|
+ pub queries: HashMap<String, Query>,
|
|
|
}
|
|
|
|
|
|
impl Account {
|
|
|
-
|
|
|
-
|
|
|
- pub fn run_queries(&self) -> HashMap<String, HashMap<String, Auction>> {
|
|
|
+ pub fn run_queries(&self) -> HashMap<String, Query> {
|
|
|
self.queries
|
|
|
- .clone()
|
|
|
- .into_iter()
|
|
|
- .map(|(url, mut query)| (url.clone(), {
|
|
|
- query.auctions.extend(auctions_from_url(&url));
|
|
|
- query.avg_price = query.get_avg_price();
|
|
|
- query
|
|
|
- }))
|
|
|
- .map(|(url, query)| {
|
|
|
-
|
|
|
- (url, query.auctions
|
|
|
+ .clone()
|
|
|
.into_iter()
|
|
|
- .map(|(u, mut a)| {
|
|
|
- if Utc::now().with_timezone(&Local).timestamp() > a.timestamp {
|
|
|
- // auction has expired, mark price as final
|
|
|
- a.is_price_final = true;
|
|
|
- }
|
|
|
- (u,a)
|
|
|
- }).collect())
|
|
|
- })
|
|
|
- .collect()
|
|
|
+ .map(|(url, mut query)| {
|
|
|
+ (url.clone(), {
|
|
|
+ query.auctions.extend(auctions_from_url(&url));
|
|
|
+ query.avg_price = query.get_avg_price();
|
|
|
+ query
|
|
|
+ })
|
|
|
+ })
|
|
|
+ .map(|(url, mut query)| {
|
|
|
+ query.auctions = query.detect_expired();
|
|
|
+
|
|
|
+ (url, query)
|
|
|
+ })
|
|
|
+ .collect()
|
|
|
}
|
|
|
|
|
|
-
|
|
|
- // pub fn run_query(&self) -> HashMap<String, Query> {
|
|
|
- // self.queries
|
|
|
- // .clone()
|
|
|
- // .into_iter()
|
|
|
- // .map(|(url, mut auction_map)| (url.clone(), {
|
|
|
- // auction_map.extend(auctions_from_url(&url));
|
|
|
- // auction_map
|
|
|
- // }))
|
|
|
- // .map(|(url, mut auction_map)| {
|
|
|
-
|
|
|
- // (url, auction_map.into_iter().map(|(u, mut a)| {
|
|
|
- // if Utc::now().with_timezone(&Local).timestamp() > a.timestamp {
|
|
|
- // // auction has expired, mark price as final
|
|
|
- // a.is_price_final = true;
|
|
|
- // }
|
|
|
- // (u, Query{ auctions: HashMap::new()})
|
|
|
- // }).collect())
|
|
|
- // })
|
|
|
- // .collect()
|
|
|
- // }
|
|
|
-
|
|
|
pub fn updated(&self) -> Account {
|
|
|
Account {
|
|
|
- // queries: self.run_queries(),
|
|
|
- queries: self.run_queries()
|
|
|
- .iter()
|
|
|
- .map(|(k,v)| (k.clone(), Query{auctions: v.clone(), .. Default::default()}))
|
|
|
- .collect()
|
|
|
-
|
|
|
- }
|
|
|
+ queries: self.run_queries(), // .iter()
|
|
|
+ // .map(|(k,v)| (k.clone(), Query{auctions: v.clone(), .. Default::default()}))
|
|
|
+ // .collect()
|
|
|
+ }
|
|
|
}
|
|
|
-
|
|
|
}
|
|
|
|
|
|
-// #[derive(Serialize, Deserialize, Debug, Clone)]
|
|
|
-// pub struct Query {
|
|
|
-// pub url: String,
|
|
|
-// pub auctions: Vec<Auction>,
|
|
|
-// pub frequency: i64,
|
|
|
-// }
|
|
|
-
|
|
|
-// impl Query {
|
|
|
-// pub fn run(&mut self) -> Query {
|
|
|
-// self.auctions = parse_url(&self.url);
|
|
|
-// self.clone()
|
|
|
-// }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-// fn detect_frequency(&self) {
|
|
|
-// let _a = &self.auctions
|
|
|
-// .iter()
|
|
|
-// .map(|x| x)
|
|
|
-// .collect::<Vec<_>>();
|
|
|
-// }
|
|
|
-// }
|
|
|
-
|
|
|
-// impl Default for Query {
|
|
|
-// fn default () -> Query {
|
|
|
-// Query{
|
|
|
-// frequency: Duration::minutes(5).num_seconds(),
|
|
|
-// url: "".to_string(),
|
|
|
-// auctions: vec![]
|
|
|
-// }
|
|
|
-// }
|
|
|
-// }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
fn date_to_gcal(date: DateTime<Local>) -> String {
|
|
|
- format!("{}",
|
|
|
- date.format("%Y%m%dT%H%M00/%Y%m%dT%H%M00")
|
|
|
- )
|
|
|
+ format!("{}", date.format("%Y%m%dT%H%M00/%Y%m%dT%H%M00"))
|
|
|
}
|
|
|
|
|
|
-
|
|
|
/// Parse a relative offset "x Tage HH:MM" into proper time
|
|
|
fn parse_remaining(timestring: &String) -> Option<Duration> {
|
|
|
- match parse(&format!("{} minutes", timestring.replace(":", " hours ").replace("Tage", "days"))).ok() {
|
|
|
+ match parse(&format!(
|
|
|
+ "{} minutes",
|
|
|
+ timestring.replace(":", " hours ").replace("Tage", "days")
|
|
|
+ ))
|
|
|
+ .ok()
|
|
|
+ {
|
|
|
Some(old_duration) => Duration::from_std(old_duration).ok(),
|
|
|
- None => None
|
|
|
+ None => None,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-
|
|
|
fn parse_price(price: &str) -> Option<f32> {
|
|
|
- price.to_string()
|
|
|
- .replace(".", "")
|
|
|
- .replace(",", ".")
|
|
|
- .replace(" EUR", "")
|
|
|
- .parse().ok()
|
|
|
+ price
|
|
|
+ .to_string()
|
|
|
+ .replace(".", "")
|
|
|
+ .replace(",", ".")
|
|
|
+ .replace(" EUR", "")
|
|
|
+ .parse()
|
|
|
+ .ok()
|
|
|
}
|
|
|
|
|
|
|
|
|
-fn parse_url(url: &str) -> Vec<Auction> {
|
|
|
- let mut auctions = vec![];
|
|
|
- // dbg!(&url);
|
|
|
-
|
|
|
- if let Ok(mut resp) = reqwest::get(url) {
|
|
|
-
|
|
|
- if !resp.status().is_success() {
|
|
|
- println!("ERR {:?}", resp.text());
|
|
|
- return auctions;
|
|
|
- }
|
|
|
-
|
|
|
- let text = resp.text().unwrap_or("".to_string());
|
|
|
-
|
|
|
|
|
|
+fn auctions_from_url(url: &str) -> HashMap<String, Auction> {
|
|
|
+ let mut auctions = HashMap::new();
|
|
|
+ dbg!(&url);
|
|
|
|
|
|
- for node in Document::from_read(text.as_bytes())
|
|
|
- .unwrap()
|
|
|
- // .find(Name("a"))
|
|
|
- // .filter(|n| n.attr("href").is_some())
|
|
|
- // .filter(|n| n.attr("href").unwrap().contains("item.php?id="))
|
|
|
- .find(Name("tr"))
|
|
|
- .filter(|x| x.attr("bgcolor").is_some())
|
|
|
- .filter(|x| x.attr("align").is_none())
|
|
|
+ let client = reqwest::Client::builder()
|
|
|
+ .timeout(Some(core::time::Duration::from_secs(2)))
|
|
|
+ .build()
|
|
|
+ .unwrap();
|
|
|
|
|
|
- {
|
|
|
|
|
|
- // Get auction name
|
|
|
- if let Some(name) = node
|
|
|
- .children().into_iter()
|
|
|
- .filter(|x| x.name() == Some("td"))
|
|
|
- .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .filter(|x| x.name() == Some("a"))
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .map(|x| x.text())
|
|
|
- // .filter(|x| x.name() == Some("Text"))
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
+ match client.get(url).send() {
|
|
|
+ Ok(mut resp) => {
|
|
|
+ if !resp.status().is_success() {
|
|
|
+ // println!("ERR {:?}", resp.text_with_charset("windows-1252"));
|
|
|
+
|
|
|
+ return auctions;
|
|
|
+ }
|
|
|
|
|
|
+ // dbg!(&resp.text_with_charset("windows-1252"));
|
|
|
+ // dbg!(&resp.text());
|
|
|
+ let text = resp.text().unwrap_or("".to_string());
|
|
|
+ // let text = resp.text_with_charset("windows-1252").unwrap_or("".to_string());
|
|
|
+
|
|
|
+ for node in Document::from_read(text.as_bytes())
|
|
|
+ .unwrap()
|
|
|
+ // .find(Name("a"))
|
|
|
+ // .filter(|n| n.attr("href").is_some())
|
|
|
+ // .filter(|n| n.attr("href").unwrap().contains("item.php?id="))
|
|
|
+ .find(Name("tr"))
|
|
|
+ .filter(|x| x.attr("bgcolor").is_some())
|
|
|
+ .filter(|x| x.attr("align").is_none())
|
|
|
+ {
|
|
|
+ // Get auction name
|
|
|
+ if let Some(name) = node
|
|
|
+ .children()
|
|
|
+ .into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
+ .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| x.name() == Some("a"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .map(|x| x.text())
|
|
|
+ // .filter(|x| x.name() == Some("Text"))
|
|
|
+ .collect::<Vec<_>>()
|
|
|
+ .get(0)
|
|
|
+ {
|
|
|
// If we have the name, go on finding other details
|
|
|
// instantiate mutable Auction
|
|
|
let mut auction = Auction::default();
|
|
|
auction.desc = name.clone();
|
|
|
auction.thumb = format!("http://egun.de/market/images/picture.gif");
|
|
|
|
|
|
-
|
|
|
- // get image
|
|
|
+ // get image
|
|
|
if let Some(img) = node
|
|
|
- .children().into_iter()
|
|
|
- .filter(|x| x.name() == Some("td"))
|
|
|
- .filter(|x| x.attr("align") == Some("center"))
|
|
|
- .flat_map(|x| x.descendants())
|
|
|
- .filter(|x| x.name() == Some("img"))
|
|
|
- .filter(|x| match x.attr("src") {
|
|
|
- Some(src) => src.contains("cache"),
|
|
|
- None => false
|
|
|
- } )
|
|
|
- .map(|x| x.attr("src").unwrap()) // we just tested
|
|
|
-
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
- auction.thumb = format!("http://egun.de/market/{}", img);
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- // get price
|
|
|
- if let Some(price) = node
|
|
|
- .children()
|
|
|
- .filter(|x| x.text().contains("EUR"))
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .map(|x| parse_price(&x.text()))
|
|
|
- .flat_map(|x| x)
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
- auction.price = price.clone();
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- // get article url
|
|
|
- if let Some(url) = node
|
|
|
- .children().into_iter()
|
|
|
- .filter(|x| x.name() == Some("td"))
|
|
|
- .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .filter(|x| x.name() == Some("a"))
|
|
|
- .map(|x| x.attr("href"))
|
|
|
- .filter_map(|x| x)
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
- auction.url = format!("http://egun.de/market/{}", url);
|
|
|
- }
|
|
|
-
|
|
|
- // TODO: check if https://doc.rust-lang.org/std/time/struct.SystemTime.html works too
|
|
|
- if let Some(remaining) = parse_remaining(
|
|
|
- &node
|
|
|
.children()
|
|
|
+ .into_iter()
|
|
|
+ .filter(|x| x.name() == Some("td"))
|
|
|
.filter(|x| x.attr("align") == Some("center"))
|
|
|
- .filter(|x| x.attr("nowrap").is_some())
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .filter(|x| !x.text().is_empty())
|
|
|
- .map(|x| x.text())
|
|
|
+ .flat_map(|x| x.descendants())
|
|
|
+ .filter(|x| x.name() == Some("img"))
|
|
|
+ .filter(|x| match x.attr("src") {
|
|
|
+ Some(src) => src.contains("cache"),
|
|
|
+ None => false,
|
|
|
+ })
|
|
|
+ .map(|x| x.attr("src").unwrap()) // we just tested
|
|
|
.collect::<Vec<_>>()
|
|
|
- .join(" ")
|
|
|
- ){
|
|
|
- // dbg!(&t_remaining.children());
|
|
|
-
|
|
|
- let end_date = Utc::now().with_timezone(&Local) + remaining;
|
|
|
- auction.gcal = format!("http://www.google.com/calendar/event?action=TEMPLATE&dates={}&text={}&location=&details=", date_to_gcal(end_date), auction.desc);
|
|
|
- auction.remaining = remaining.num_seconds();
|
|
|
- auction.timestamp = end_date.timestamp();
|
|
|
- // println!("ENDS\t{:?}", date_to_gcal(remaining));
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- auctions.push(auction);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
- auctions
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-fn auctions_from_url(url: &str) -> HashMap<String, Auction> {
|
|
|
- let mut auctions = HashMap::new();
|
|
|
- // dbg!(&url);
|
|
|
-
|
|
|
- if let Ok(mut resp) = reqwest::get(url) {
|
|
|
-
|
|
|
- if !resp.status().is_success() {
|
|
|
- println!("ERR {:?}", resp.text());
|
|
|
- return auctions;
|
|
|
- }
|
|
|
-
|
|
|
- let text = resp.text().unwrap_or("".to_string());
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- for node in Document::from_read(text.as_bytes())
|
|
|
- .unwrap()
|
|
|
- // .find(Name("a"))
|
|
|
- // .filter(|n| n.attr("href").is_some())
|
|
|
- // .filter(|n| n.attr("href").unwrap().contains("item.php?id="))
|
|
|
- .find(Name("tr"))
|
|
|
- .filter(|x| x.attr("bgcolor").is_some())
|
|
|
- .filter(|x| x.attr("align").is_none())
|
|
|
-
|
|
|
- {
|
|
|
-
|
|
|
- // Get auction name
|
|
|
- if let Some(name) = node
|
|
|
- .children().into_iter()
|
|
|
- .filter(|x| x.name() == Some("td"))
|
|
|
- .filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .filter(|x| x.name() == Some("a"))
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .map(|x| x.text())
|
|
|
- // .filter(|x| x.name() == Some("Text"))
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
-
|
|
|
- // If we have the name, go on finding other details
|
|
|
- // instantiate mutable Auction
|
|
|
- let mut auction = Auction::default();
|
|
|
- auction.desc = name.clone();
|
|
|
- auction.thumb = format!("http://egun.de/market/images/picture.gif");
|
|
|
-
|
|
|
-
|
|
|
- // get image
|
|
|
- if let Some(img) = node
|
|
|
- .children().into_iter()
|
|
|
- .filter(|x| x.name() == Some("td"))
|
|
|
- .filter(|x| x.attr("align") == Some("center"))
|
|
|
- .flat_map(|x| x.descendants())
|
|
|
- .filter(|x| x.name() == Some("img"))
|
|
|
- .filter(|x| match x.attr("src") {
|
|
|
- Some(src) => src.contains("cache"),
|
|
|
- None => false
|
|
|
- } )
|
|
|
- .map(|x| x.attr("src").unwrap()) // we just tested
|
|
|
-
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
+ .get(0)
|
|
|
+ {
|
|
|
auction.thumb = format!("http://egun.de/market/{}", img);
|
|
|
}
|
|
|
|
|
|
-
|
|
|
- // get price
|
|
|
+ // get price
|
|
|
if let Some(price) = node
|
|
|
- .children()
|
|
|
- .filter(|x| x.text().contains("EUR"))
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .map(|x| parse_price(&x.text()))
|
|
|
- .flat_map(|x| x)
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
+ .children()
|
|
|
+ .filter(|x| x.text().contains("EUR"))
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .map(|x| parse_price(&x.text()))
|
|
|
+ .flat_map(|x| x)
|
|
|
+ .collect::<Vec<_>>()
|
|
|
+ .get(0)
|
|
|
+ {
|
|
|
auction.price = price.clone();
|
|
|
}
|
|
|
|
|
|
-
|
|
|
// get article url
|
|
|
if let Some(url) = node
|
|
|
- .children().into_iter()
|
|
|
+ .children()
|
|
|
+ .into_iter()
|
|
|
.filter(|x| x.name() == Some("td"))
|
|
|
.filter(|x| x.attr("align") == Some("LEFT"))
|
|
|
.flat_map(|x| x.children())
|
|
|
.filter(|x| x.name() == Some("a"))
|
|
|
.map(|x| x.attr("href"))
|
|
|
.filter_map(|x| x)
|
|
|
- .collect::<Vec<_>>().get(0) {
|
|
|
+ .collect::<Vec<_>>()
|
|
|
+ .get(0)
|
|
|
+ {
|
|
|
auction.url = format!("http://egun.de/market/{}", url);
|
|
|
}
|
|
|
-
|
|
|
// TODO: check if https://doc.rust-lang.org/std/time/struct.SystemTime.html works too
|
|
|
if let Some(remaining) = parse_remaining(
|
|
|
&node
|
|
|
- .children()
|
|
|
- .filter(|x| x.attr("align") == Some("center"))
|
|
|
- .filter(|x| x.attr("nowrap").is_some())
|
|
|
- .flat_map(|x| x.children())
|
|
|
- .filter(|x| !x.text().is_empty())
|
|
|
- .map(|x| x.text())
|
|
|
- .collect::<Vec<_>>()
|
|
|
- .join(" ")
|
|
|
- ){
|
|
|
+ .children()
|
|
|
+ .filter(|x| x.attr("align") == Some("center"))
|
|
|
+ .filter(|x| x.attr("nowrap").is_some())
|
|
|
+ .flat_map(|x| x.children())
|
|
|
+ .filter(|x| !x.text().is_empty())
|
|
|
+ .map(|x| x.text())
|
|
|
+ .collect::<Vec<_>>()
|
|
|
+ .join(" "),
|
|
|
+ ) {
|
|
|
// dbg!(&t_remaining.children());
|
|
|
|
|
|
let end_date = Utc::now().with_timezone(&Local) + remaining;
|
|
|
@@ -392,13 +232,16 @@ fn auctions_from_url(url: &str) -> HashMap<String, Auction> {
|
|
|
auction.remaining = remaining.num_seconds();
|
|
|
auction.timestamp = end_date.timestamp();
|
|
|
// println!("ENDS\t{:?}", date_to_gcal(remaining));
|
|
|
-
|
|
|
}
|
|
|
|
|
|
auctions.insert(auction.url.clone(), auction);
|
|
|
}
|
|
|
+ }
|
|
|
}
|
|
|
-
|
|
|
+ Err(e) => println!("{:?}", e),
|
|
|
}
|
|
|
+
|
|
|
+ // if let Ok(mut resp) = client.get(url).send() {
|
|
|
+
|
|
|
auctions
|
|
|
-}
|
|
|
+}
|