Improved getting market data performance

Core/Main/PTMagic.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Threading;
 using System.IO;
 using System.Linq;
@@ -52,7 +53,7 @@ namespace Core.Main
     private List<string> _indicatorsLines = null;
     private List<string> _exchangeMarketList = null;
     private List<string> _marketList = new List<string>();
-    private Dictionary<string, MarketInfo> _marketInfos = new Dictionary<string, MarketInfo>();
+    private ConcurrentDictionary<string, MarketInfo> _marketInfos = new ConcurrentDictionary<string, MarketInfo>();
     private Dictionary<string, double> _averageMarketTrendChanges = new Dictionary<string, double>();
     private Dictionary<string, List<MarketTrendChange>> _singleMarketTrendChanges = new Dictionary<string, List<MarketTrendChange>>();
     private Dictionary<string, List<MarketTrendChange>> _globalMarketTrendChanges = new Dictionary<string, List<MarketTrendChange>>();
@@ -432,7 +433,7 @@ namespace Core.Main
       }
     }
 
-    public Dictionary<string, MarketInfo> MarketInfos
+    public ConcurrentDictionary<string, MarketInfo> MarketInfos
     {
       get
       {
@@ -903,6 +904,28 @@ namespace Core.Main
               // Check for single market trend triggers
               this.ApplySingleMarketSettings();
 
+              // Ignore quarterly futures
+              if (this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceFutures", StringComparison.InvariantCultureIgnoreCase))
+              {
+                // Find all quarterly futures pairs
+                var results = this.MarketList.FindAll(m => m.Contains("_", StringComparison.InvariantCultureIgnoreCase));
+
+                // Create the settings lines to disable trading
+                if (results.Count > 0)
+                {
+                  this.PairsLines.AddRange(new string[] {
+                    "",
+                    "# Binance Futures quarterly futures ignore list",
+                    "###############################################"
+                  });
+                
+                  foreach (var marketPair in results)
+                  {
+                    this.PairsLines.Add(String.Format("{0}_trading_enabled = false", marketPair));
+                  }
+                }
+              }
+
               // Save new properties to Profit Trailer
               this.SaveProfitTrailerProperties();
 
@@ -1518,7 +1541,7 @@ namespace Core.Main
 
             // Check ignore markets
             List<string> ignoredMarkets = SystemHelper.ConvertTokenStringToList(marketSetting.IgnoredMarkets, ",");
-            if (ignoredMarkets.Contains(marketPair))
+            if (ignoredMarkets.Any(im => marketPair.StartsWith(im, StringComparison.InvariantCultureIgnoreCase)))
             {
               this.Log.DoLogDebug("'" + marketPair + "' - Is ignored in '" + marketSetting.SettingName + "'.");
               continue;
@@ -1526,7 +1549,7 @@ namespace Core.Main
 
             // Check allowed markets
             List<string> allowedMarkets = SystemHelper.ConvertTokenStringToList(marketSetting.AllowedMarkets, ",");
-            if (allowedMarkets.Count > 0 && !allowedMarkets.Contains(marketPair))
+            if (allowedMarkets.Count > 0 && !allowedMarkets.Any(am => marketPair.StartsWith(am, StringComparison.InvariantCultureIgnoreCase)))
             {
               this.Log.DoLogDebug("'" + marketPair + "' - Is not allowed in '" + marketSetting.SettingName + "'.");
               continue;

Core/MarketAnalyzer/BaseAnalyzer.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.IO;
 using System.Linq;
 using System.Net;
@@ -153,16 +154,16 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static Dictionary<string, MarketInfo> GetMarketInfosFromFile(PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static ConcurrentDictionary<string, MarketInfo> GetMarketInfosFromFile(PTMagicConfiguration systemConfiguration, LogHelper log)
     {
-      Dictionary<string, MarketInfo> result = new Dictionary<string, MarketInfo>();
+      ConcurrentDictionary<string, MarketInfo> result = new ConcurrentDictionary<string, MarketInfo>();
 
       string marketInfoFilePath = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar + "MarketInfo.json";
       if (File.Exists(marketInfoFilePath))
       {
         try
         {
-          result = JsonConvert.DeserializeObject<Dictionary<string, MarketInfo>>(System.IO.File.ReadAllText(marketInfoFilePath));
+          result = JsonConvert.DeserializeObject<ConcurrentDictionary<string, MarketInfo>>(System.IO.File.ReadAllText(marketInfoFilePath));
         }
         catch (Exception ex)
         {
@@ -171,12 +172,12 @@ namespace Core.MarketAnalyzer
       }
       if (result == null)
       {
-        result = new Dictionary<string, MarketInfo>();
+        result = new ConcurrentDictionary<string, MarketInfo>();
       }
       return result;
     }
 
-    public static void SaveMarketInfosToFile(Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static void SaveMarketInfosToFile(ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketInfo.json", JsonConvert.SerializeObject(marketInfos));
     }

Core/MarketAnalyzer/Binance.cs

@@ -1,17 +1,14 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.IO;
-using System.Text;
 using Core.Main;
 using Core.Helper;
 using Core.Main.DataObjects.PTMagicData;
 using Newtonsoft.Json;
-using Core.ProfitTrailer;
 using System.Net;
-using System.Threading;
 using System.Threading.Tasks;
-using System.Collections.Concurrent;
 
 namespace Core.MarketAnalyzer
 {
@@ -43,7 +40,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       List<string> result = new List<string>();
 
@@ -163,7 +160,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       log.DoLogInfo("Binance - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
 
@@ -182,7 +179,7 @@ namespace Core.MarketAnalyzer
         {
           marketInfo = new MarketInfo();
           marketInfo.Name = key;
-          marketInfos.Add(key, marketInfo);
+          marketInfos.TryAdd(key, marketInfo);
           marketInfo.FirstSeen = Binance.GetFirstSeenDate(key, systemConfiguration, log);
         }
         else
@@ -375,7 +372,7 @@ namespace Core.MarketAnalyzer
         }
 
         Parallel.ForEach(markets.Keys,
-                          new ParallelOptions { MaxDegreeOfParallelism = ParallelThrottle},
+                          new ParallelOptions { MaxDegreeOfParallelism = ParallelThrottle },
                           (key) =>
         {
           if (!marketTicks.TryAdd(key, GetMarketTicks(key, totalTicks, systemConfiguration, log)))
@@ -413,8 +410,10 @@ namespace Core.MarketAnalyzer
               }
             }
 
-            Dictionary<string, Market> tickMarkets = new Dictionary<string, Market>();
+            ConcurrentDictionary<string, Market> tickMarkets = new ConcurrentDictionary<string, Market>();
-            foreach (string key in markets.Keys)
+
+            Parallel.ForEach(markets.Keys,
+                          (key) =>
                             {
                               List<MarketTick> tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List<MarketTick>();
 
@@ -430,9 +429,9 @@ namespace Core.MarketAnalyzer
                                 //market.Volume24h = marketTick.Volume24h;
                                 market.MainCurrencyPriceUSD = mainCurrencyPrice;
 
-                tickMarkets.Add(market.Name, market);
+                                tickMarkets.TryAdd(market.Name, market);
-              }
                               }
+                            });
 
             DateTime fileDateTime = new DateTime(tickTime.ToLocalTime().Year, tickTime.ToLocalTime().Month, tickTime.ToLocalTime().Day, tickTime.ToLocalTime().Hour, tickTime.ToLocalTime().Minute, 0).ToUniversalTime();
 

Core/MarketAnalyzer/BinanceFutures.cs

@@ -1,17 +1,14 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.IO;
-using System.Text;
 using Core.Main;
 using Core.Helper;
 using Core.Main.DataObjects.PTMagicData;
 using Newtonsoft.Json;
-using Core.ProfitTrailer;
 using System.Net;
-using System.Threading;
 using System.Threading.Tasks;
-using System.Collections.Concurrent;
 
 namespace Core.MarketAnalyzer
 {
@@ -43,7 +40,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       List<string> result = new List<string>();
 
@@ -163,7 +160,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       log.DoLogInfo("BinanceFutures - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
 
@@ -182,7 +179,7 @@ namespace Core.MarketAnalyzer
         {
           marketInfo = new MarketInfo();
           marketInfo.Name = key;
-          marketInfos.Add(key, marketInfo);
+          marketInfos.TryAdd(key, marketInfo);
           marketInfo.FirstSeen = BinanceFutures.GetFirstSeenDate(key, systemConfiguration, log);
         }
         else

Core/MarketAnalyzer/BinanceUS.cs

@@ -2,14 +2,11 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.IO;
-using System.Text;
 using Core.Main;
 using Core.Helper;
 using Core.Main.DataObjects.PTMagicData;
 using Newtonsoft.Json;
-using Core.ProfitTrailer;
 using System.Net;
-using System.Threading;
 using System.Threading.Tasks;
 using System.Collections.Concurrent;
 
@@ -48,7 +45,7 @@ namespace Core.MarketAnalyzer
       }
     }
 
-    public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       List<string> result = new List<string>();
 
@@ -168,7 +165,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       log.DoLogInfo("BinanceUS - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
 
@@ -187,7 +184,7 @@ namespace Core.MarketAnalyzer
         {
           marketInfo = new MarketInfo();
           marketInfo.Name = key;
-          marketInfos.Add(key, marketInfo);
+          marketInfos.TryAdd(key, marketInfo);
           marketInfo.FirstSeen = BinanceUS.GetFirstSeenDate(key, systemConfiguration, log);
         }
         else

Core/MarketAnalyzer/Bittrex.cs

@@ -1,14 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.IO;
-using System.Text;
 using Core.Main;
 using Core.Helper;
 using Core.Main.DataObjects.PTMagicData;
 using Newtonsoft.Json;
-using Core.ProfitTrailer;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Core.MarketAnalyzer
@@ -44,7 +42,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       List<string> result = new List<string>();
 
@@ -104,7 +102,7 @@ namespace Core.MarketAnalyzer
                   {
                     marketInfo = new MarketInfo();
                     marketInfo.Name = marketName;
-                    marketInfos.Add(marketName, marketInfo);
+                    marketInfos.TryAdd(marketName, marketInfo);
                   }
                   if (currencyTicker["Summary"]["Created"].Type == Newtonsoft.Json.Linq.JTokenType.Date) marketInfo.FirstSeen = (DateTime)currencyTicker["Summary"]["Created"];
                   marketInfo.LastSeen = DateTime.UtcNow;

Core/MarketAnalyzer/Poloniex.cs

@@ -1,14 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.IO;
-using System.Text;
 using Core.Main;
 using Core.Helper;
 using Core.Main.DataObjects.PTMagicData;
 using Newtonsoft.Json;
-using Core.ProfitTrailer;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Core.MarketAnalyzer
@@ -44,7 +42,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       List<string> result = new List<string>();
 
@@ -125,7 +123,7 @@ namespace Core.MarketAnalyzer
       return result;
     }
 
-    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
     {
       log.DoLogInfo("Poloniex - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
 
@@ -142,7 +140,7 @@ namespace Core.MarketAnalyzer
         {
           marketInfo = new MarketInfo();
           marketInfo.Name = key;
-          marketInfos.Add(key, marketInfo);
+          marketInfos.TryAdd(key, marketInfo);
           marketInfo.FirstSeen = Poloniex.GetFirstSeenDate(key, systemConfiguration, log);
         }
         else

Monitor/Pages/_get/BagDetails.cshtml

@@ -103,7 +103,7 @@
       <div class="tradingview-widget-container">
         <div id="tradingview_6aa22" style="height:600px;"></div>
         <div class="tradingview-widget-copyright">
-          <a href="@Core.Helper.SystemHelper.GetMarketLink(Model.PTMagicConfiguration.GeneralSettings.Monitor.LinkPlatform,Model.PTMagicConfiguration.GeneralSettings.Application.Exchange, Model.DCAMarket, Model.Summary.MainMarket)" rel="noopener" target="_blank">
+          <a href="@Core.Helper.SystemHelper.GetMarketLink(Model.PTMagicConfiguration.GeneralSettings.Monitor.LinkPlatform, Model.PTMagicConfiguration.GeneralSettings.Application.Exchange, Model.DCAMarket, Model.Summary.MainMarket)" rel="noopener" target="_blank">
           <span class="blue-text">@Model.DCAMarket</span> <span class="blue-text">chart</span> by TradingView</a>
         </div>
       </div>