trading
/
PTMagic
mirror of https://github.com/PTMagicians/PTMagic.git
2
0
Fork 0
Code Issues Releases Wiki Activity

Improved getting market data performance

This commit is contained in:
djbadders 2021-06-20 11:34:37 +01:00
parent e7d1f246ce
commit 8249e5e182
8 changed files with 72 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
Core/Main/PTMagic.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Threading;
using System.IO;
using System.Linq;
@ -52,7 +53,7 @@ namespace Core.Main
private List<string> _indicatorsLines = null;
private List<string> _exchangeMarketList = null;
private List<string> _marketList = new List<string>();
private Dictionary<string, MarketInfo> _marketInfos = new Dictionary<string, MarketInfo>();
private ConcurrentDictionary<string, MarketInfo> _marketInfos = new ConcurrentDictionary<string, MarketInfo>();
private Dictionary<string, double> _averageMarketTrendChanges = new Dictionary<string, double>();
private Dictionary<string, List<MarketTrendChange>> _singleMarketTrendChanges = new Dictionary<string, List<MarketTrendChange>>();
private Dictionary<string, List<MarketTrendChange>> _globalMarketTrendChanges = new Dictionary<string, List<MarketTrendChange>>();
@ -432,7 +433,7 @@ namespace Core.Main
}
}
public Dictionary<string, MarketInfo> MarketInfos
public ConcurrentDictionary<string, MarketInfo> MarketInfos
{
get
{
@ -903,6 +904,28 @@ namespace Core.Main
// Check for single market trend triggers
this.ApplySingleMarketSettings();
// Ignore quarterly futures
if (this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceFutures", StringComparison.InvariantCultureIgnoreCase))
{
// Find all quarterly futures pairs
var results = this.MarketList.FindAll(m => m.Contains("_", StringComparison.InvariantCultureIgnoreCase));
// Create the settings lines to disable trading
if (results.Count > 0)
{
this.PairsLines.AddRange(new string[] {
"",
"# Binance Futures quarterly futures ignore list",
"###############################################"
});
foreach (var marketPair in results)
{
this.PairsLines.Add(String.Format("{0}_trading_enabled = false", marketPair));
}
}
}
// Save new properties to Profit Trailer
this.SaveProfitTrailerProperties();
@ -1518,7 +1541,7 @@ namespace Core.Main
// Check ignore markets
List<string> ignoredMarkets = SystemHelper.ConvertTokenStringToList(marketSetting.IgnoredMarkets, ",");
if (ignoredMarkets.Contains(marketPair))
if (ignoredMarkets.Any(im => marketPair.StartsWith(im, StringComparison.InvariantCultureIgnoreCase)))
{
this.Log.DoLogDebug("'" + marketPair + "' - Is ignored in '" + marketSetting.SettingName + "'.");
continue;
@ -1526,7 +1549,7 @@ namespace Core.Main
// Check allowed markets
List<string> allowedMarkets = SystemHelper.ConvertTokenStringToList(marketSetting.AllowedMarkets, ",");
if (allowedMarkets.Count > 0 && !allowedMarkets.Contains(marketPair))
if (allowedMarkets.Count > 0 && !allowedMarkets.Any(am => marketPair.StartsWith(am, StringComparison.InvariantCultureIgnoreCase)))
{
this.Log.DoLogDebug("'" + marketPair + "' - Is not allowed in '" + marketSetting.SettingName + "'.");
continue;

11
Core/MarketAnalyzer/BaseAnalyzer.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Net;
@ -153,16 +154,16 @@ namespace Core.MarketAnalyzer
return result;
}
public static Dictionary<string, MarketInfo> GetMarketInfosFromFile(PTMagicConfiguration systemConfiguration, LogHelper log)
public static ConcurrentDictionary<string, MarketInfo> GetMarketInfosFromFile(PTMagicConfiguration systemConfiguration, LogHelper log)
{
Dictionary<string, MarketInfo> result = new Dictionary<string, MarketInfo>();
ConcurrentDictionary<string, MarketInfo> result = new ConcurrentDictionary<string, MarketInfo>();
string marketInfoFilePath = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar + "MarketInfo.json";
if (File.Exists(marketInfoFilePath))
{
try
{
result = JsonConvert.DeserializeObject<Dictionary<string, MarketInfo>>(System.IO.File.ReadAllText(marketInfoFilePath));
result = JsonConvert.DeserializeObject<ConcurrentDictionary<string, MarketInfo>>(System.IO.File.ReadAllText(marketInfoFilePath));
}
catch (Exception ex)
{
@ -171,12 +172,12 @@ namespace Core.MarketAnalyzer
}
if (result == null)
{
result = new Dictionary<string, MarketInfo>();
result = new ConcurrentDictionary<string, MarketInfo>();
}
return result;
}
public static void SaveMarketInfosToFile(Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static void SaveMarketInfosToFile(ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketInfo.json", JsonConvert.SerializeObject(marketInfos));
}

49
Core/MarketAnalyzer/Binance.cs
View File

@ -1,17 +1,14 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Linq;
using System.IO;
using System.Text;
using Core.Main;
using Core.Helper;
using Core.Main.DataObjects.PTMagicData;
using Newtonsoft.Json;
using Core.ProfitTrailer;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;
namespace Core.MarketAnalyzer
{
@ -43,7 +40,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List<string> result = new List<string>();
@ -163,7 +160,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
log.DoLogInfo("Binance - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
@ -182,7 +179,7 @@ namespace Core.MarketAnalyzer
{
marketInfo = new MarketInfo();
marketInfo.Name = key;
marketInfos.Add(key, marketInfo);
marketInfos.TryAdd(key, marketInfo);
marketInfo.FirstSeen = Binance.GetFirstSeenDate(key, systemConfiguration, log);
}
else
@ -375,7 +372,7 @@ namespace Core.MarketAnalyzer
}
Parallel.ForEach(markets.Keys,
new ParallelOptions { MaxDegreeOfParallelism = ParallelThrottle},
new ParallelOptions { MaxDegreeOfParallelism = ParallelThrottle },
(key) =>
{
if (!marketTicks.TryAdd(key, GetMarketTicks(key, totalTicks, systemConfiguration, log)))
@ -413,26 +410,28 @@ namespace Core.MarketAnalyzer
}
}
Dictionary<string, Market> tickMarkets = new Dictionary<string, Market>();
foreach (string key in markets.Keys)
{
List<MarketTick> tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List<MarketTick>();
ConcurrentDictionary<string, Market> tickMarkets = new ConcurrentDictionary<string, Market>();
if (tickRange.Count > 0)
{
MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First();
Parallel.ForEach(markets.Keys,
(key) =>
{
List<MarketTick> tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List<MarketTick>();
Market market = new Market();
market.Position = markets.Count + 1;
market.Name = key;
market.Symbol = key;
market.Price = marketTick.Price;
//market.Volume24h = marketTick.Volume24h;
market.MainCurrencyPriceUSD = mainCurrencyPrice;
if (tickRange.Count > 0)
{
MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First();
tickMarkets.Add(market.Name, market);
}
}
Market market = new Market();
market.Position = markets.Count + 1;
market.Name = key;
market.Symbol = key;
market.Price = marketTick.Price;
//market.Volume24h = marketTick.Volume24h;
market.MainCurrencyPriceUSD = mainCurrencyPrice;
tickMarkets.TryAdd(market.Name, market);
}
});
DateTime fileDateTime = new DateTime(tickTime.ToLocalTime().Year, tickTime.ToLocalTime().Month, tickTime.ToLocalTime().Day, tickTime.ToLocalTime().Hour, tickTime.ToLocalTime().Minute, 0).ToUniversalTime();

11
Core/MarketAnalyzer/BinanceFutures.cs
View File

@ -1,17 +1,14 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Linq;
using System.IO;
using System.Text;
using Core.Main;
using Core.Helper;
using Core.Main.DataObjects.PTMagicData;
using Newtonsoft.Json;
using Core.ProfitTrailer;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;
namespace Core.MarketAnalyzer
{
@ -43,7 +40,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List<string> result = new List<string>();
@ -163,7 +160,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
log.DoLogInfo("BinanceFutures - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
@ -182,7 +179,7 @@ namespace Core.MarketAnalyzer
{
marketInfo = new MarketInfo();
marketInfo.Name = key;
marketInfos.Add(key, marketInfo);
marketInfos.TryAdd(key, marketInfo);
marketInfo.FirstSeen = BinanceFutures.GetFirstSeenDate(key, systemConfiguration, log);
}
else

9
Core/MarketAnalyzer/BinanceUS.cs
View File

@ -2,14 +2,11 @@
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.Text;
using Core.Main;
using Core.Helper;
using Core.Main.DataObjects.PTMagicData;
using Newtonsoft.Json;
using Core.ProfitTrailer;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;
@ -48,7 +45,7 @@ namespace Core.MarketAnalyzer
}
}
public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List<string> result = new List<string>();
@ -168,7 +165,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
log.DoLogInfo("BinanceUS - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
@ -187,7 +184,7 @@ namespace Core.MarketAnalyzer
{
marketInfo = new MarketInfo();
marketInfo.Name = key;
marketInfos.Add(key, marketInfo);
marketInfos.TryAdd(key, marketInfo);
marketInfo.FirstSeen = BinanceUS.GetFirstSeenDate(key, systemConfiguration, log);
}
else

8
Core/MarketAnalyzer/Bittrex.cs
View File

@ -1,14 +1,12 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Linq;
using System.IO;
using System.Text;
using Core.Main;
using Core.Helper;
using Core.Main.DataObjects.PTMagicData;
using Newtonsoft.Json;
using Core.ProfitTrailer;
using System.Threading;
using System.Threading.Tasks;
namespace Core.MarketAnalyzer
@ -44,7 +42,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List<string> result = new List<string>();
@ -104,7 +102,7 @@ namespace Core.MarketAnalyzer
{
marketInfo = new MarketInfo();
marketInfo.Name = marketName;
marketInfos.Add(marketName, marketInfo);
marketInfos.TryAdd(marketName, marketInfo);
}
if (currencyTicker["Summary"]["Created"].Type == Newtonsoft.Json.Linq.JTokenType.Date) marketInfo.FirstSeen = (DateTime)currencyTicker["Summary"]["Created"];
marketInfo.LastSeen = DateTime.UtcNow;

10
Core/MarketAnalyzer/Poloniex.cs
View File

@ -1,14 +1,12 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Linq;
using System.IO;
using System.Text;
using Core.Main;
using Core.Helper;
using Core.Main.DataObjects.PTMagicData;
using Newtonsoft.Json;
using Core.ProfitTrailer;
using System.Threading;
using System.Threading.Tasks;
namespace Core.MarketAnalyzer
@ -44,7 +42,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static List<string> GetMarketData(string mainMarket, Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static List<string> GetMarketData(string mainMarket, ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
List<string> result = new List<string>();
@ -125,7 +123,7 @@ namespace Core.MarketAnalyzer
return result;
}
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref Dictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
public static void CheckFirstSeenDates(Dictionary<string, Market> markets, ref ConcurrentDictionary<string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
{
log.DoLogInfo("Poloniex - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
@ -142,7 +140,7 @@ namespace Core.MarketAnalyzer
{
marketInfo = new MarketInfo();
marketInfo.Name = key;
marketInfos.Add(key, marketInfo);
marketInfos.TryAdd(key, marketInfo);
marketInfo.FirstSeen = Poloniex.GetFirstSeenDate(key, systemConfiguration, log);
}
else

2
Monitor/Pages/_get/BagDetails.cshtml
View File

@ -103,7 +103,7 @@
<div class="tradingview-widget-container">
<div id="tradingview_6aa22" style="height:600px;"></div>
<div class="tradingview-widget-copyright">
<a href="@Core.Helper.SystemHelper.GetMarketLink(Model.PTMagicConfiguration.GeneralSettings.Monitor.LinkPlatform,Model.PTMagicConfiguration.GeneralSettings.Application.Exchange, Model.DCAMarket, Model.Summary.MainMarket)" rel="noopener" target="_blank">
<a href="@Core.Helper.SystemHelper.GetMarketLink(Model.PTMagicConfiguration.GeneralSettings.Monitor.LinkPlatform, Model.PTMagicConfiguration.GeneralSettings.Application.Exchange, Model.DCAMarket, Model.Summary.MainMarket)" rel="noopener" target="_blank">
<span class="blue-text">@Model.DCAMarket</span> <span class="blue-text">chart</span> by TradingView</a>
</div>
</div>