respect max connections

src/main.rs

@@ -33,28 +33,37 @@ lazy_static! {
 
 fn daemon() {
     loop {
-        info!(">>> Running update...");
+        info!(">>> UPD");
 
         let accounts_unlocked = ACCOUNTS.lock().unwrap();
         let accounts_cl = accounts_unlocked.clone();
         drop(accounts_unlocked);
 
+        let client = reqwest::blocking::Client::builder()
+        // .max_idle_per_host(200)
+        // .timeout(Some(core::time::Duration::from_secs(10)))
+        .build()
+        .unwrap();
+
         let mut threads = vec![];
 
         for (account_name, account) in accounts_cl {
+            let client = client.clone();
             let t = thread::spawn(move || {
-                let updated_account = account.updated();
+                info!("ACCT {}", account_name);
+                
+                let updated_account = account.updated(&client);
                 let mut accounts = ACCOUNTS.lock().unwrap();
                 accounts.insert(account_name.to_string(), updated_account);
             });
             threads.push(t);
         }
 
-        info!("Joining threads");
+        debug!("Joining threads");
         for t in threads {
             t.join().unwrap();
         }
-        info!("Done.");
+        debug!("Done.");
 
 
         let accounts_unlocked = ACCOUNTS.lock().unwrap();
@@ -62,7 +71,7 @@ fn daemon() {
         serde_json::to_writer_pretty(writer, &*accounts_unlocked).unwrap();
         drop(accounts_unlocked);
 
-        info!("Cycle over, sleeping 30 secs.");
+        info!("<<< Cycle over, sleeping 30 secs.");
         let pause = time::Duration::from_secs(30);
         thread::sleep(pause);
 

src/query.rs

@@ -1,6 +1,8 @@
 use chrono::{DateTime, Duration, Local, Utc};
 use parse_duration::parse;
 use reqwest;
+use reqwest::blocking::Client;
+
 use select::document::Document;
 use select::predicate::Name;
 use std::collections::HashMap;
@@ -8,6 +10,8 @@ use std::collections::HashMap;
 use url::form_urlencoded::{byte_serialize};
 use log::*;
 
+const MAXCLIENTS: usize = 8;
+
 #[derive(Serialize, Deserialize, Debug, Default, Clone)]
 pub struct Auction {
     pub price: f32,
@@ -22,7 +26,11 @@ pub struct Auction {
 
 
 impl Auction {
-    fn update(&mut self) {
+    fn update(&mut self, client: &Client) {
+
+
+
+
         // println!("now: {:?} auction: {:?}", Utc::now().with_timezone(&Local).timestamp(), self.timestamp);
         let was_not_final = self.is_price_final;
         self.is_price_final = Utc::now().with_timezone(&Local).timestamp() > self.timestamp;
@@ -32,7 +40,7 @@ impl Auction {
             dbg!(&self.price);
         } 
 
-        *self = auction_details(&self);
+        *self = auction_details(&self, client);
 
         // if was_not_final != self.is_price_final {
         //     self.price = auction_details(&self.url);
@@ -56,13 +64,16 @@ impl Query {
     fn get_avg_price(&mut self) -> f32 {
         self.auctions.iter().map(|(_u, a)| a.price).sum::<f32>() / self.auctions.len() as f32
     }
+    
+
+
 
-    fn detect_expired(&self) -> HashMap<String, Auction> {
+    fn detect_expired(&self, client: &Client) -> HashMap<String, Auction> {
         self.auctions
             .iter()
             .map(|(u, a)| {
                 let mut a1 = a.clone();
-                a1.update();
+                a1.update(client);
                 (u.clone(), a1)
             })
             .collect()
@@ -77,13 +88,13 @@ pub struct Account {
 }
 
 impl Account {
-    pub fn run_queries(&self) -> HashMap<String, Query> {
+    pub fn run_queries(&self, client: &Client) -> HashMap<String, Query> {
         self.queries
             .clone()
             .into_iter()
             .map(|(url, mut query)| {
                 (url.clone(), {
-                    query.auctions.extend(auctions_from_url(&url));
+                    query.auctions.extend(auctions_from_url(&url, client));
                     query.avg_price = query.get_avg_price();
                     if cfg!(debug_assertions) {
                         // dbg!(&query);
@@ -92,15 +103,15 @@ impl Account {
                 })
             })
             .map(|(url, mut query)| {
-                query.auctions = query.detect_expired();
+                query.auctions = query.detect_expired(client);
                 (url, query)
             })
             .collect()
     }
 
-    pub fn updated(&self) -> Account {
+    pub fn updated(&self, client: &Client) -> Account {
         Account {
-            queries: self.run_queries(), // .iter()
+            queries: self.run_queries(client), // .iter()
                                          // .map(|(k,v)| (k.clone(), Query{auctions: v.clone(), .. Default::default()}))
                                          // .collect()
         }
@@ -137,17 +148,18 @@ fn parse_price(price: &str) -> Option<f32> {
 }
 
 
-fn auction_details(auction: &Auction) -> Auction {
+fn auction_details(auction: &Auction, client: &Client) -> Auction {
     let mut updated_auction = auction.clone();
 
     if auction.is_price_final {
         return updated_auction;
     }
 
-    let client = reqwest::blocking::Client::builder()
-        // .timeout(Some(core::time::Duration::from_secs(10)))
-        .build()
-        .unwrap();
+    // let client = reqwest::blocking::Client::builder()
+    //     .max_idle_per_host(8)
+    //     // .timeout(Some(core::time::Duration::from_secs(10)))
+    //     .build()
+    //     .unwrap();
 
 
     match client.get(&auction.url).send() {
@@ -202,14 +214,11 @@ fn abs_image_url(rel_url: &str) -> String {
 }
 
 
-fn auctions_from_url(url: &str) -> HashMap<String, Auction> {
+fn auctions_from_url(url: &str, client: &Client) -> HashMap<String, Auction> {
     let mut auctions = HashMap::new();
     // dbg!(&url);
 
-    let client = reqwest::blocking::Client::builder()
-        // .timeout(Some(core::time::Duration::from_secs(8)))
-        .build()
-        .unwrap();
+ 
 
     match client.get(url).send() {
         Ok(mut resp) => {