guntrader/src/main.rs

#[macro_use]
extern crate serde_derive;
use reqwest;
use chrono::{Utc, Duration, Local, DateTime};
use select::document::Document;
use select::predicate::Name;
use parse_duration::parse;
use serde;
use serde_json;
use std::io::BufWriter;
use std::io::BufReader;
use std::fs::File;

/// Fucking egun is a mess. It does not even have classes. This is an attempt to parse it.



#[derive(Serialize, Deserialize, Debug, Default)]
struct Auction {
    price: f32,
    desc: String,
    gcal: String,
    thumb: String,
    remaining: String,
    url: String,
    timestamp: i64,
    is_price_final: bool
}

#[derive(Serialize, Deserialize, Debug)]
struct Query {
    url: String,
    auctions: Vec<Auction>,
    frequency: i64
}


impl Query {
    fn run(&mut self) {
        self.auctions = parse_url(&self.url);
    }
}

impl Default for Query {
    fn default () -> Query {
        Query{
            frequency: Duration::minutes(5).num_seconds(),
            url: "".to_string(),
            auctions: vec![]
        }
    }
}


#[derive(Serialize, Deserialize, Debug, Default)]
struct Account {
    queries: Vec<Query>,
    name: String,
}



fn date_to_gcal(date: DateTime<Local>) -> String {
    format!("{}",
        date.format("%Y%m%dT%H%M00/%Y%m%dT%H%M00")
    )
}

/// Parse a relative offset "x Tage HH:MM" into proper time
fn parse_end_date(timestring: &String) -> Option<DateTime<Local>> {
    match parse(&format!("{} minutes", timestring.replace(":", " hours ").replace("Tage", "days"))).ok() {
        Some(old_duration) => match Duration::from_std(old_duration).ok() {
            Some(chronoduration) => Some(Utc::now().with_timezone(&Local) + chronoduration),
            None => None
        }
        None => None
    }
}

fn parse_price(price: &str) -> Option<f32> {
    price.to_string()
    .replace(".", "")
    .replace(",", ".")
    .replace(" EUR", "")
    .parse().ok()
}


fn parse_url(url: &str) -> Vec<Auction> {
    let mut resp = reqwest::get(url).unwrap();
    assert!(resp.status().is_success());
    // dbg!(&resp.text());

    let text = resp.text().unwrap();

    let mut auctions = vec![];


    for node in Document::from_read(text.as_bytes())
        .unwrap()
        // .find(Name("a"))
        // .filter(|n| n.attr("href").is_some())
        // .filter(|n| n.attr("href").unwrap().contains("item.php?id="))
        .find(Name("tr"))
        .filter(|x| x.attr("bgcolor").is_some())
        .filter(|x| x.attr("align").is_none())

        {

            // Get auction name
            if let Some(name) = node
                .children().into_iter()
                .filter(|x| x.name() == Some("td"))
                .filter(|x| x.attr("align") == Some("LEFT"))
                .flat_map(|x| x.children())
                .filter(|x| x.name() == Some("a"))
                .flat_map(|x| x.children())
                .map(|x| x.text())
                // .filter(|x| x.name() == Some("Text"))
                .collect::<Vec<_>>().get(0) {

                    // If we have the name, go on finding other details
                    // instantiate mutable Auction
                    let mut auction = Auction::default();
                    auction.desc = name.clone();

                    // get image                
                    if let Some(img) = node
                    .children().into_iter()
                    .filter(|x| x.name() == Some("td"))
                    .filter(|x| x.attr("align") == Some("center"))
                    .flat_map(|x| x.descendants())
                    .filter(|x| x.name() == Some("img"))
                    .filter(|x| match x.attr("src") {
                        Some(src) => src.contains("cache"),
                        None => false
                    }  )
                    .map(|x| x.attr("src").unwrap()) // we just tested

                    .collect::<Vec<_>>().get(0) {
                        auction.thumb = format!("http://egun.de/market/{}", img);
                    }


                    // get price                
                    if let Some(price) = node
                    .children()
                    .filter(|x| x.text().contains("EUR"))
                    .flat_map(|x| x.children())
                    .map(|x| parse_price(&x.text()))
                    .flat_map(|x| x)
                    .collect::<Vec<_>>().get(0) {
                        auction.price = price.clone();
                    }


                
                    // get article url
                    if let Some(url) = node
                    .children().into_iter()
                        .filter(|x| x.name() == Some("td"))
                        .filter(|x| x.attr("align") == Some("LEFT"))
                        .flat_map(|x| x.children())
                        .filter(|x| x.name() == Some("a"))
                        .map(|x| x.attr("href"))
                        .filter_map(|x| x)
                    .collect::<Vec<_>>().get(0) {
                        auction.url = format!("http://egun.de/market/{}", url);
                    }
                
                    // TODO: check if https://doc.rust-lang.org/std/time/struct.SystemTime.html works too
                    if let Some(remaining) = parse_end_date(
                        &node
                        .children()
                        .filter(|x| x.attr("align") == Some("center"))
                        .filter(|x| x.attr("nowrap").is_some())
                        .flat_map(|x| x.children())
                        .filter(|x| !x.text().is_empty())
                        .map(|x| x.text())
                        .collect::<Vec<_>>()
                        .join(" ")
                    ){
                        // dbg!(&t_remaining.children());
                        auction.gcal = format!("http://www.google.com/calendar/event?action=TEMPLATE&dates={}&text={}&location=&details=", date_to_gcal(remaining), auction.desc);
                        auction.timestamp = remaining.timestamp();
                        // println!("ENDS\t{:?}", date_to_gcal(remaining));

                    }

                    auctions.push(auction);
                }
        }

        auctions
}




fn main() {
    let url = "http://www.egun.de/market/list_items.php?mode=qry&plusdescr=off&wheremode=and&query=sniper&quick=1";
    

    let mut query = Query {
        url: url.to_string(),
        ..Default::default()
    };

    query.run();
    dbg!(query);

    // write out the file
    let writer = BufWriter::new(File::create("db.json").unwrap());
    serde_json::to_writer_pretty(writer, &parse_url(url)).unwrap();
}