From 8249e5e1825b474c617a396b23fea3d48a2bbed3 Mon Sep 17 00:00:00 2001 From: djbadders <34887832+djbadders@users.noreply.github.com> Date: Sun, 20 Jun 2021 11:34:37 +0100 Subject: [PATCH] Improved getting market data performance --- Core/Main/PTMagic.cs | 31 ++++++++++++++--- Core/MarketAnalyzer/BaseAnalyzer.cs | 11 +++--- Core/MarketAnalyzer/Binance.cs | 49 +++++++++++++-------------- Core/MarketAnalyzer/BinanceFutures.cs | 11 +++--- Core/MarketAnalyzer/BinanceUS.cs | 9 ++--- Core/MarketAnalyzer/Bittrex.cs | 8 ++--- Core/MarketAnalyzer/Poloniex.cs | 10 +++--- Monitor/Pages/_get/BagDetails.cshtml | 2 +- 8 files changed, 72 insertions(+), 59 deletions(-) diff --git a/Core/Main/PTMagic.cs b/Core/Main/PTMagic.cs index c585351..405f2eb 100644 --- a/Core/Main/PTMagic.cs +++ b/Core/Main/PTMagic.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Concurrent; using System.Threading; using System.IO; using System.Linq; @@ -52,7 +53,7 @@ namespace Core.Main private List _indicatorsLines = null; private List _exchangeMarketList = null; private List _marketList = new List(); - private Dictionary _marketInfos = new Dictionary(); + private ConcurrentDictionary _marketInfos = new ConcurrentDictionary(); private Dictionary _averageMarketTrendChanges = new Dictionary(); private Dictionary> _singleMarketTrendChanges = new Dictionary>(); private Dictionary> _globalMarketTrendChanges = new Dictionary>(); @@ -432,7 +433,7 @@ namespace Core.Main } } - public Dictionary MarketInfos + public ConcurrentDictionary MarketInfos { get { @@ -903,6 +904,28 @@ namespace Core.Main // Check for single market trend triggers this.ApplySingleMarketSettings(); + // Ignore quarterly futures + if (this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceFutures", StringComparison.InvariantCultureIgnoreCase)) + { + // Find all quarterly futures pairs + var results = this.MarketList.FindAll(m => m.Contains("_", StringComparison.InvariantCultureIgnoreCase)); + + // Create the settings lines to disable trading + if (results.Count > 0) + { + this.PairsLines.AddRange(new string[] { + "", + "# Binance Futures quarterly futures ignore list", + "###############################################" + }); + + foreach (var marketPair in results) + { + this.PairsLines.Add(String.Format("{0}_trading_enabled = false", marketPair)); + } + } + } + // Save new properties to Profit Trailer this.SaveProfitTrailerProperties(); @@ -1518,7 +1541,7 @@ namespace Core.Main // Check ignore markets List ignoredMarkets = SystemHelper.ConvertTokenStringToList(marketSetting.IgnoredMarkets, ","); - if (ignoredMarkets.Contains(marketPair)) + if (ignoredMarkets.Any(im => marketPair.StartsWith(im, StringComparison.InvariantCultureIgnoreCase))) { this.Log.DoLogDebug("'" + marketPair + "' - Is ignored in '" + marketSetting.SettingName + "'."); continue; @@ -1526,7 +1549,7 @@ namespace Core.Main // Check allowed markets List allowedMarkets = SystemHelper.ConvertTokenStringToList(marketSetting.AllowedMarkets, ","); - if (allowedMarkets.Count > 0 && !allowedMarkets.Contains(marketPair)) + if (allowedMarkets.Count > 0 && !allowedMarkets.Any(am => marketPair.StartsWith(am, StringComparison.InvariantCultureIgnoreCase))) { this.Log.DoLogDebug("'" + marketPair + "' - Is not allowed in '" + marketSetting.SettingName + "'."); continue; diff --git a/Core/MarketAnalyzer/BaseAnalyzer.cs b/Core/MarketAnalyzer/BaseAnalyzer.cs index 51b2628..5c1b66e 100644 --- a/Core/MarketAnalyzer/BaseAnalyzer.cs +++ b/Core/MarketAnalyzer/BaseAnalyzer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Concurrent; using System.IO; using System.Linq; using System.Net; @@ -153,16 +154,16 @@ namespace Core.MarketAnalyzer return result; } - public static Dictionary GetMarketInfosFromFile(PTMagicConfiguration systemConfiguration, LogHelper log) + public static ConcurrentDictionary GetMarketInfosFromFile(PTMagicConfiguration systemConfiguration, LogHelper log) { - Dictionary result = new Dictionary(); + ConcurrentDictionary result = new ConcurrentDictionary(); string marketInfoFilePath = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar + "MarketInfo.json"; if (File.Exists(marketInfoFilePath)) { try { - result = JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(marketInfoFilePath)); + result = JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(marketInfoFilePath)); } catch (Exception ex) { @@ -171,12 +172,12 @@ namespace Core.MarketAnalyzer } if (result == null) { - result = new Dictionary(); + result = new ConcurrentDictionary(); } return result; } - public static void SaveMarketInfosToFile(Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static void SaveMarketInfosToFile(ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketInfo.json", JsonConvert.SerializeObject(marketInfos)); } diff --git a/Core/MarketAnalyzer/Binance.cs b/Core/MarketAnalyzer/Binance.cs index 8eae0a7..f6f0bc5 100644 --- a/Core/MarketAnalyzer/Binance.cs +++ b/Core/MarketAnalyzer/Binance.cs @@ -1,17 +1,14 @@ using System; using System.Collections.Generic; +using System.Collections.Concurrent; using System.Linq; using System.IO; -using System.Text; using Core.Main; using Core.Helper; using Core.Main.DataObjects.PTMagicData; using Newtonsoft.Json; -using Core.ProfitTrailer; using System.Net; -using System.Threading; using System.Threading.Tasks; -using System.Collections.Concurrent; namespace Core.MarketAnalyzer { @@ -43,7 +40,7 @@ namespace Core.MarketAnalyzer return result; } - public static List GetMarketData(string mainMarket, Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static List GetMarketData(string mainMarket, ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { List result = new List(); @@ -163,7 +160,7 @@ namespace Core.MarketAnalyzer return result; } - public static void CheckFirstSeenDates(Dictionary markets, ref Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static void CheckFirstSeenDates(Dictionary markets, ref ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { log.DoLogInfo("Binance - Checking first seen dates for " + markets.Count + " markets. This may take a while..."); @@ -182,7 +179,7 @@ namespace Core.MarketAnalyzer { marketInfo = new MarketInfo(); marketInfo.Name = key; - marketInfos.Add(key, marketInfo); + marketInfos.TryAdd(key, marketInfo); marketInfo.FirstSeen = Binance.GetFirstSeenDate(key, systemConfiguration, log); } else @@ -375,7 +372,7 @@ namespace Core.MarketAnalyzer } Parallel.ForEach(markets.Keys, - new ParallelOptions { MaxDegreeOfParallelism = ParallelThrottle}, + new ParallelOptions { MaxDegreeOfParallelism = ParallelThrottle }, (key) => { if (!marketTicks.TryAdd(key, GetMarketTicks(key, totalTicks, systemConfiguration, log))) @@ -413,26 +410,28 @@ namespace Core.MarketAnalyzer } } - Dictionary tickMarkets = new Dictionary(); - foreach (string key in markets.Keys) - { - List tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List(); + ConcurrentDictionary tickMarkets = new ConcurrentDictionary(); - if (tickRange.Count > 0) - { - MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First(); + Parallel.ForEach(markets.Keys, + (key) => + { + List tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List(); - Market market = new Market(); - market.Position = markets.Count + 1; - market.Name = key; - market.Symbol = key; - market.Price = marketTick.Price; - //market.Volume24h = marketTick.Volume24h; - market.MainCurrencyPriceUSD = mainCurrencyPrice; + if (tickRange.Count > 0) + { + MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First(); - tickMarkets.Add(market.Name, market); - } - } + Market market = new Market(); + market.Position = markets.Count + 1; + market.Name = key; + market.Symbol = key; + market.Price = marketTick.Price; + //market.Volume24h = marketTick.Volume24h; + market.MainCurrencyPriceUSD = mainCurrencyPrice; + + tickMarkets.TryAdd(market.Name, market); + } + }); DateTime fileDateTime = new DateTime(tickTime.ToLocalTime().Year, tickTime.ToLocalTime().Month, tickTime.ToLocalTime().Day, tickTime.ToLocalTime().Hour, tickTime.ToLocalTime().Minute, 0).ToUniversalTime(); diff --git a/Core/MarketAnalyzer/BinanceFutures.cs b/Core/MarketAnalyzer/BinanceFutures.cs index de1f962..5a02643 100644 --- a/Core/MarketAnalyzer/BinanceFutures.cs +++ b/Core/MarketAnalyzer/BinanceFutures.cs @@ -1,17 +1,14 @@ using System; using System.Collections.Generic; +using System.Collections.Concurrent; using System.Linq; using System.IO; -using System.Text; using Core.Main; using Core.Helper; using Core.Main.DataObjects.PTMagicData; using Newtonsoft.Json; -using Core.ProfitTrailer; using System.Net; -using System.Threading; using System.Threading.Tasks; -using System.Collections.Concurrent; namespace Core.MarketAnalyzer { @@ -43,7 +40,7 @@ namespace Core.MarketAnalyzer return result; } - public static List GetMarketData(string mainMarket, Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static List GetMarketData(string mainMarket, ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { List result = new List(); @@ -163,7 +160,7 @@ namespace Core.MarketAnalyzer return result; } - public static void CheckFirstSeenDates(Dictionary markets, ref Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static void CheckFirstSeenDates(Dictionary markets, ref ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { log.DoLogInfo("BinanceFutures - Checking first seen dates for " + markets.Count + " markets. This may take a while..."); @@ -182,7 +179,7 @@ namespace Core.MarketAnalyzer { marketInfo = new MarketInfo(); marketInfo.Name = key; - marketInfos.Add(key, marketInfo); + marketInfos.TryAdd(key, marketInfo); marketInfo.FirstSeen = BinanceFutures.GetFirstSeenDate(key, systemConfiguration, log); } else diff --git a/Core/MarketAnalyzer/BinanceUS.cs b/Core/MarketAnalyzer/BinanceUS.cs index 63b1318..0b09180 100644 --- a/Core/MarketAnalyzer/BinanceUS.cs +++ b/Core/MarketAnalyzer/BinanceUS.cs @@ -2,14 +2,11 @@ using System.Collections.Generic; using System.Linq; using System.IO; -using System.Text; using Core.Main; using Core.Helper; using Core.Main.DataObjects.PTMagicData; using Newtonsoft.Json; -using Core.ProfitTrailer; using System.Net; -using System.Threading; using System.Threading.Tasks; using System.Collections.Concurrent; @@ -48,7 +45,7 @@ namespace Core.MarketAnalyzer } } - public static List GetMarketData(string mainMarket, Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static List GetMarketData(string mainMarket, ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { List result = new List(); @@ -168,7 +165,7 @@ namespace Core.MarketAnalyzer return result; } - public static void CheckFirstSeenDates(Dictionary markets, ref Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static void CheckFirstSeenDates(Dictionary markets, ref ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { log.DoLogInfo("BinanceUS - Checking first seen dates for " + markets.Count + " markets. This may take a while..."); @@ -187,7 +184,7 @@ namespace Core.MarketAnalyzer { marketInfo = new MarketInfo(); marketInfo.Name = key; - marketInfos.Add(key, marketInfo); + marketInfos.TryAdd(key, marketInfo); marketInfo.FirstSeen = BinanceUS.GetFirstSeenDate(key, systemConfiguration, log); } else diff --git a/Core/MarketAnalyzer/Bittrex.cs b/Core/MarketAnalyzer/Bittrex.cs index c7ede44..e5470e1 100644 --- a/Core/MarketAnalyzer/Bittrex.cs +++ b/Core/MarketAnalyzer/Bittrex.cs @@ -1,14 +1,12 @@ using System; using System.Collections.Generic; +using System.Collections.Concurrent; using System.Linq; using System.IO; -using System.Text; using Core.Main; using Core.Helper; using Core.Main.DataObjects.PTMagicData; using Newtonsoft.Json; -using Core.ProfitTrailer; -using System.Threading; using System.Threading.Tasks; namespace Core.MarketAnalyzer @@ -44,7 +42,7 @@ namespace Core.MarketAnalyzer return result; } - public static List GetMarketData(string mainMarket, Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static List GetMarketData(string mainMarket, ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { List result = new List(); @@ -104,7 +102,7 @@ namespace Core.MarketAnalyzer { marketInfo = new MarketInfo(); marketInfo.Name = marketName; - marketInfos.Add(marketName, marketInfo); + marketInfos.TryAdd(marketName, marketInfo); } if (currencyTicker["Summary"]["Created"].Type == Newtonsoft.Json.Linq.JTokenType.Date) marketInfo.FirstSeen = (DateTime)currencyTicker["Summary"]["Created"]; marketInfo.LastSeen = DateTime.UtcNow; diff --git a/Core/MarketAnalyzer/Poloniex.cs b/Core/MarketAnalyzer/Poloniex.cs index 84a5848..64f29aa 100644 --- a/Core/MarketAnalyzer/Poloniex.cs +++ b/Core/MarketAnalyzer/Poloniex.cs @@ -1,14 +1,12 @@ using System; using System.Collections.Generic; +using System.Collections.Concurrent; using System.Linq; using System.IO; -using System.Text; using Core.Main; using Core.Helper; using Core.Main.DataObjects.PTMagicData; using Newtonsoft.Json; -using Core.ProfitTrailer; -using System.Threading; using System.Threading.Tasks; namespace Core.MarketAnalyzer @@ -44,7 +42,7 @@ namespace Core.MarketAnalyzer return result; } - public static List GetMarketData(string mainMarket, Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static List GetMarketData(string mainMarket, ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { List result = new List(); @@ -125,7 +123,7 @@ namespace Core.MarketAnalyzer return result; } - public static void CheckFirstSeenDates(Dictionary markets, ref Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + public static void CheckFirstSeenDates(Dictionary markets, ref ConcurrentDictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) { log.DoLogInfo("Poloniex - Checking first seen dates for " + markets.Count + " markets. This may take a while..."); @@ -142,7 +140,7 @@ namespace Core.MarketAnalyzer { marketInfo = new MarketInfo(); marketInfo.Name = key; - marketInfos.Add(key, marketInfo); + marketInfos.TryAdd(key, marketInfo); marketInfo.FirstSeen = Poloniex.GetFirstSeenDate(key, systemConfiguration, log); } else diff --git a/Monitor/Pages/_get/BagDetails.cshtml b/Monitor/Pages/_get/BagDetails.cshtml index 087a4e1..2c5e4f8 100644 --- a/Monitor/Pages/_get/BagDetails.cshtml +++ b/Monitor/Pages/_get/BagDetails.cshtml @@ -103,7 +103,7 @@
- + @Model.DCAMarket chart by TradingView