diff --git a/Core/Helper/SystemHelper.cs b/Core/Helper/SystemHelper.cs index 5d6c4e3..398b403 100644 --- a/Core/Helper/SystemHelper.cs +++ b/Core/Helper/SystemHelper.cs @@ -560,6 +560,9 @@ namespace Core.Helper case "BinanceUS": result = "https://www.binance.us/trade.html?symbol=" + market; break; + case "BinanceFutures": + result = "https://www.binance.com/en/futures/" + market; + break; case "Poloniex": result = "https://poloniex.com/exchange#" + market.ToLower(); break; @@ -584,6 +587,9 @@ namespace Core.Helper case "BinanceUS": result = market + mainMarket; break; + case "BinanceFutures": + result = market + "_" + mainMarket; + break; case "Poloniex": result = mainMarket + "_" + market; break; diff --git a/Core/Main/PTMagic.cs b/Core/Main/PTMagic.cs index 181d7e7..305b49c 100644 --- a/Core/Main/PTMagic.cs +++ b/Core/Main/PTMagic.cs @@ -691,6 +691,7 @@ namespace Core.Main if (!this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Binance", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Bittrex", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceUS", StringComparison.InvariantCultureIgnoreCase) + && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceFutures", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Poloniex", StringComparison.InvariantCultureIgnoreCase)) { this.Log.DoLogError("Exchange '" + this.PTMagicConfiguration.GeneralSettings.Application.Exchange + "' specified in settings.general.json is invalid! Terminating process..."); @@ -1087,7 +1088,7 @@ namespace Core.Main } else { - if (!this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Binance", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceUS", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Bittrex", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Poloniex", StringComparison.InvariantCultureIgnoreCase)) + if (!this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Binance", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceUS", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceFutures", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Bittrex", StringComparison.InvariantCultureIgnoreCase) && !this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Poloniex", StringComparison.InvariantCultureIgnoreCase)) { Log.DoLogError("Your setting for Application.Exchange in settings.general.json is invalid (" + this.PTMagicConfiguration.GeneralSettings.Application.Exchange + ")! Terminating process."); this.Timer.Stop(); @@ -1246,6 +1247,11 @@ namespace Core.Main // Get most recent market data from BinanceUS this.ExchangeMarketList = BinanceUS.GetMarketData(this.LastRuntimeSummary.MainMarket, this.MarketInfos, this.PTMagicConfiguration, this.Log); } + else if (this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("BinanceFutures", StringComparison.InvariantCultureIgnoreCase)) + { + // Get most recent market data from BinanceFutures + this.ExchangeMarketList = BinanceFutures.GetMarketData(this.LastRuntimeSummary.MainMarket, this.MarketInfos, this.PTMagicConfiguration, this.Log); + } else if (this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Poloniex", StringComparison.InvariantCultureIgnoreCase)) { diff --git a/Core/MarketAnalyzer/BaseAnalyzer.cs b/Core/MarketAnalyzer/BaseAnalyzer.cs index e865d53..2865926 100644 --- a/Core/MarketAnalyzer/BaseAnalyzer.cs +++ b/Core/MarketAnalyzer/BaseAnalyzer.cs @@ -50,7 +50,7 @@ namespace Core.MarketAnalyzer try { - log.DoLogInfo("Calling URL: " + url); + // log.DoLogInfo("Calling URL: " + url); var response = webClient.GetAsync(url).Result; string repsonseString = response.Content.ReadAsStringAsync().Result; diff --git a/Core/MarketAnalyzer/BinanceFutures.cs b/Core/MarketAnalyzer/BinanceFutures.cs new file mode 100644 index 0000000..de1f962 --- /dev/null +++ b/Core/MarketAnalyzer/BinanceFutures.cs @@ -0,0 +1,455 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Text; +using Core.Main; +using Core.Helper; +using Core.Main.DataObjects.PTMagicData; +using Newtonsoft.Json; +using Core.ProfitTrailer; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Concurrent; + +namespace Core.MarketAnalyzer +{ + public class BinanceFutures : BaseAnalyzer + { + public static double GetMainCurrencyPrice(string mainMarket, PTMagicConfiguration systemConfiguration, LogHelper log) + { + double result = 0; + + try + { + string baseUrl = "https://fapi.binance.com/fapi/v1/ticker/24hr?symbol=" + mainMarket + "USDT"; + + log.DoLogInfo("BinanceFutures - Getting main market price..."); + Newtonsoft.Json.Linq.JObject jsonObject = GetSimpleJsonObjectFromURL(baseUrl, log, null); + if (jsonObject != null) + { + log.DoLogInfo("BinanceFutures - Market data received for " + mainMarket + "USDT"); + + result = (double)jsonObject.GetValue("lastPrice"); + log.DoLogInfo("BinanceFutures - Current price for " + mainMarket + "USDT: " + result.ToString("#,#0.00") + " USD"); + } + } + catch (Exception ex) + { + log.DoLogCritical(ex.Message, ex); + } + + return result; + } + + public static List GetMarketData(string mainMarket, Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + { + List result = new List(); + + string lastMarket = ""; + Newtonsoft.Json.Linq.JObject lastTicker = null; + try + { + string baseUrl = "https://fapi.binance.com/fapi/v1/ticker/24hr"; + + log.DoLogInfo("BinanceFutures - Getting market data..."); + Newtonsoft.Json.Linq.JArray jsonArray = GetSimpleJsonArrayFromURL(baseUrl, log); + if (jsonArray.Count > 0) + { + double mainCurrencyPrice = 1; + if (!mainMarket.Equals("USDT", StringComparison.InvariantCultureIgnoreCase)) + { + mainCurrencyPrice = BinanceFutures.GetMainCurrencyPrice(mainMarket, systemConfiguration, log); + } + + log.DoLogInfo("BinanceFutures - Market data received for " + jsonArray.Count.ToString() + " currencies"); + + if (mainCurrencyPrice > 0) + { + Dictionary markets = new Dictionary(); + foreach (Newtonsoft.Json.Linq.JObject currencyTicker in jsonArray) + { + string marketName = currencyTicker["symbol"].ToString(); + //New variables for filtering out bad markets + float marketLastPrice = currencyTicker["lastPrice"].ToObject(); + float marketVolume = currencyTicker["volume"].ToObject(); + if (marketName.EndsWith(mainMarket, StringComparison.InvariantCultureIgnoreCase)) + { + if (marketLastPrice > 0 && marketVolume > 0) + { + + // Set last values in case any error occurs + lastMarket = marketName; + lastTicker = currencyTicker; + + Market market = new Market(); + market.Position = markets.Count + 1; + market.Name = marketName; + market.Symbol = currencyTicker["symbol"].ToString(); + market.Price = SystemHelper.TextToDouble(currencyTicker["lastPrice"].ToString(), 0, "en-US"); + market.Volume24h = SystemHelper.TextToDouble(currencyTicker["quoteVolume"].ToString(), 0, "en-US"); + market.MainCurrencyPriceUSD = mainCurrencyPrice; + + markets.Add(market.Name, market); + + result.Add(market.Name); + } + else + { + //Let the user know that the problem market was ignored. + log.DoLogInfo("BinanceFutures - Ignoring bad market data for " + marketName); + } + } + } + + BinanceFutures.CheckFirstSeenDates(markets, ref marketInfos, systemConfiguration, log); + + BaseAnalyzer.SaveMarketInfosToFile(marketInfos, systemConfiguration, log); + + BinanceFutures.CheckForMarketDataRecreation(mainMarket, markets, systemConfiguration, log); + + DateTime fileDateTime = DateTime.UtcNow; + + FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(markets), fileDateTime, fileDateTime); + + log.DoLogInfo("BinanceFutures - Market data saved for " + markets.Count.ToString() + " markets with " + mainMarket + "."); + + FileHelper.CleanupFiles(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours); + log.DoLogInfo("BinanceFutures - Market data cleaned."); + } + else + { + log.DoLogError("BinanceFutures - Failed to get main market price for " + mainMarket + "."); + result = null; + } + } + } + catch (WebException ex) + { + if (ex.Response != null) + { + using (HttpWebResponse errorResponse = (HttpWebResponse)ex.Response) + { + using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream())) + { + Dictionary errorData = JsonConvert.DeserializeObject>(reader.ReadToEnd()); + if (errorData != null) + { + string errorMessage = "Unable to get data from BinanceFutures with URL '" + errorResponse.ResponseUri + "'!"; + if (errorData.ContainsKey("code")) + { + errorMessage += " - Code: " + errorData["code"]; + } + + if (errorData.ContainsKey("msg")) + { + errorMessage += " - Message: " + errorData["msg"]; + } + + log.DoLogError(errorMessage); + } + } + } + } + result = null; + } + catch (Exception ex) + { + log.DoLogCritical("Exception while getting data for '" + lastMarket + "': " + ex.Message, ex); + result = null; + } + + return result; + } + + public static void CheckFirstSeenDates(Dictionary markets, ref Dictionary marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log) + { + log.DoLogInfo("BinanceFutures - Checking first seen dates for " + markets.Count + " markets. This may take a while..."); + + int marketsChecked = 0; + + foreach (string key in markets.Keys) + { + // Save market info + MarketInfo marketInfo = null; + if (marketInfos.ContainsKey(key)) + { + marketInfo = marketInfos[key]; + } + + if (marketInfo == null) + { + marketInfo = new MarketInfo(); + marketInfo.Name = key; + marketInfos.Add(key, marketInfo); + marketInfo.FirstSeen = BinanceFutures.GetFirstSeenDate(key, systemConfiguration, log); + } + else + { + if (marketInfo.FirstSeen == Constants.confMinDate) + { + marketInfo.FirstSeen = BinanceFutures.GetFirstSeenDate(key, systemConfiguration, log); + } + } + marketInfo.LastSeen = DateTime.UtcNow; + + marketsChecked++; + + if ((marketsChecked % 20) == 0) + { + log.DoLogInfo("BinanceFutures - Yes, I am still checking first seen dates... " + marketsChecked + "/" + markets.Count + " markets done..."); + } + } + } + + public static DateTime GetFirstSeenDate(string marketName, PTMagicConfiguration systemConfiguration, LogHelper log) + { + DateTime result = Constants.confMinDate; + + string baseUrl = "https://fapi.binance.com/fapi/v1/klines?interval=1d&symbol=" + marketName + "&limit=100"; + + log.DoLogDebug("BinanceFutures - Getting first seen date for '" + marketName + "'..."); + + Newtonsoft.Json.Linq.JArray jsonArray = GetSimpleJsonArrayFromURL(baseUrl, log); + if (jsonArray.Count > 0) + { + result = Constants.Epoch.AddMilliseconds((Int64)jsonArray[0][0]); + log.DoLogDebug("BinanceFutures - First seen date for '" + marketName + "' set to " + result.ToString()); + } + + return result; + } + + public static List GetMarketTicks(string marketName, int ticksNeeded, PTMagicConfiguration systemConfiguration, LogHelper log) + { + List result = new List(); + + try + { + Int64 endTime = (Int64)Math.Ceiling(DateTime.UtcNow.Subtract(Constants.Epoch).TotalMilliseconds); + int ticksLimit = 500; + string baseUrl = ""; + int ticksFetched = 0; + + if (ticksNeeded < ticksLimit) + { + ticksLimit = ticksNeeded; + } + + bool go = true; + while (ticksFetched < ticksNeeded && go) + { + baseUrl = "https://fapi.binance.com/fapi/v1/klines?interval=1m&symbol=" + marketName + "&endTime=" + endTime.ToString() + "&limit=" + ticksLimit.ToString(); + + log.DoLogDebug("BinanceFutures - Getting " + ticksLimit.ToString() + " ticks for '" + marketName + "'..."); + Newtonsoft.Json.Linq.JArray jsonArray = GetSimpleJsonArrayFromURL(baseUrl, log); + if (jsonArray.Count > 0) + { + log.DoLogDebug("BinanceFutures - " + jsonArray.Count.ToString() + " ticks received."); + + foreach (Newtonsoft.Json.Linq.JArray marketTick in jsonArray) + { + + MarketTick tick = new MarketTick(); + tick.Price = (double)marketTick[4]; + tick.Volume24h = (double)marketTick[7]; + tick.Time = Constants.Epoch.AddMilliseconds((Int64)marketTick[0]); + + result.Add(tick); + } + + ticksFetched = ticksFetched + jsonArray.Count; + endTime = endTime - ticksLimit * 60 * 1000; + if (ticksNeeded - ticksFetched < ticksLimit) + { + ticksLimit = ticksNeeded - ticksFetched; + } + } + else + { + log.DoLogDebug("BinanceFutures - No ticks received."); + go = false; + } + } + } + catch (WebException ex) + { + if (ex.Response != null) + { + using (HttpWebResponse errorResponse = (HttpWebResponse)ex.Response) + { + using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream())) + { + Dictionary errorData = JsonConvert.DeserializeObject>(reader.ReadToEnd()); + if (errorData != null) + { + string errorMessage = "Unable to get data from BinanceFutures with URL '" + errorResponse.ResponseUri + "'!"; + if (errorData.ContainsKey("code")) + { + errorMessage += " - Code: " + errorData["code"]; + } + + if (errorData.ContainsKey("msg")) + { + errorMessage += " - Message: " + errorData["msg"]; + } + + log.DoLogError(errorMessage); + } + } + } + } + result = null; + } + catch (Exception ex) + { + log.DoLogCritical(ex.Message, ex); + } + + return result; + } + + public static void CheckForMarketDataRecreation(string mainMarket, Dictionary markets, PTMagicConfiguration systemConfiguration, LogHelper log) + { + string binanceFuturesDataDirectoryPath = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar; + + if (!Directory.Exists(binanceFuturesDataDirectoryPath)) + { + Directory.CreateDirectory(binanceFuturesDataDirectoryPath); + } + + DirectoryInfo dataDirectory = new DirectoryInfo(binanceFuturesDataDirectoryPath); + + // Check for existing market files + DateTime latestMarketDataFileDateTime = Constants.confMinDate; + List marketFiles = dataDirectory.EnumerateFiles("MarketData*").ToList(); + FileInfo latestMarketDataFile = null; + if (marketFiles.Count > 0) + { + latestMarketDataFile = marketFiles.OrderByDescending(mdf => mdf.LastWriteTimeUtc).First(); + latestMarketDataFileDateTime = latestMarketDataFile.LastWriteTimeUtc; + } + + if (latestMarketDataFileDateTime < DateTime.UtcNow.AddMinutes(-20)) + { + int lastMarketDataAgeInSeconds = (int)Math.Ceiling(DateTime.UtcNow.Subtract(latestMarketDataFileDateTime).TotalSeconds); + + // Go back in time and create market data + DateTime startDateTime = DateTime.UtcNow; + DateTime endDateTime = DateTime.UtcNow.AddHours(-systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours); + if (latestMarketDataFileDateTime != Constants.confMinDate && latestMarketDataFileDateTime > endDateTime) + { + // Existing market files too old => Recreate market data for configured timeframe + log.DoLogInfo("BinanceFutures - Recreating market data for " + markets.Count + " markets over " + SystemHelper.GetProperDurationTime(lastMarketDataAgeInSeconds) + ". This may take a while..."); + endDateTime = latestMarketDataFileDateTime; + } + else + { + // No existing market files found => Recreate market data for configured timeframe + log.DoLogInfo("BinanceFutures - Recreating market data for " + markets.Count + " markets over " + systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours + " hours. This may take a while..."); + } + + int totalTicks = (int)Math.Ceiling(startDateTime.Subtract(endDateTime).TotalMinutes); + + // Get Ticks for main market + List mainMarketTicks = new List(); + if (!mainMarket.Equals("USDT", StringComparison.InvariantCultureIgnoreCase)) + { + mainMarketTicks = BinanceFutures.GetMarketTicks(mainMarket + "USDT", totalTicks, systemConfiguration, log); + } + + // Get Ticks for all markets + log.DoLogDebug("BinanceFutures - Getting ticks for '" + markets.Count + "' markets"); + ConcurrentDictionary> marketTicks = new ConcurrentDictionary>(); + + int ParallelThrottle = 4; + if (systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours > 200) + { + ParallelThrottle = 2; + log.DoLogInfo("----------------------------------------------------------------------------"); + log.DoLogInfo("StoreDataMaxHours is greater than 200. Historical data requests will be"); + log.DoLogInfo("throttled to avoid exceeding exchange data request limits. This initial "); + log.DoLogInfo("run could take more than 30 minutes. Please go outside for a walk..."); + log.DoLogInfo("----------------------------------------------------------------------------"); + } + + Parallel.ForEach(markets.Keys, + new ParallelOptions { MaxDegreeOfParallelism = ParallelThrottle}, + (key) => + { + if (!marketTicks.TryAdd(key, GetMarketTicks(key, totalTicks, systemConfiguration, log))) + { + // Failed to add ticks to dictionary + throw new Exception("Failed to add ticks for " + key + " to the memory dictionary, results may be incorrectly calculated!"); + } + + if ((marketTicks.Count % 10) == 0) + { + log.DoLogInfo("BinanceFutures - No worries, I am still alive... " + marketTicks.Count + "/" + markets.Count + " markets done..."); + } + }); + + log.DoLogInfo("BinanceFutures - Ticks completed."); + + log.DoLogInfo("BinanceFutures - Creating initial market data ticks. This may take another while..."); + + // Go back in time and create market data + int completedTicks = 0; + if (marketTicks.Count > 0) + { + for (DateTime tickTime = startDateTime; tickTime >= endDateTime; tickTime = tickTime.AddMinutes(-1)) + { + completedTicks++; + + double mainCurrencyPrice = 1; + if (mainMarketTicks.Count > 0) + { + List mainCurrencyTickRange = mainMarketTicks.FindAll(t => t.Time <= tickTime); + if (mainCurrencyTickRange.Count > 0) + { + MarketTick mainCurrencyTick = mainCurrencyTickRange.OrderByDescending(t => t.Time).First(); + mainCurrencyPrice = mainCurrencyTick.Price; + } + } + + Dictionary tickMarkets = new Dictionary(); + foreach (string key in markets.Keys) + { + List tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List(); + + if (tickRange.Count > 0) + { + MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First(); + + Market market = new Market(); + market.Position = markets.Count + 1; + market.Name = key; + market.Symbol = key; + market.Price = marketTick.Price; + //market.Volume24h = marketTick.Volume24h; + market.MainCurrencyPriceUSD = mainCurrencyPrice; + + tickMarkets.Add(market.Name, market); + } + } + + DateTime fileDateTime = new DateTime(tickTime.ToLocalTime().Year, tickTime.ToLocalTime().Month, tickTime.ToLocalTime().Day, tickTime.ToLocalTime().Hour, tickTime.ToLocalTime().Minute, 0).ToUniversalTime(); + + FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(tickMarkets), fileDateTime, fileDateTime); + + log.DoLogDebug("BinanceFutures - Market data saved for tick " + fileDateTime.ToString() + " - MainCurrencyPrice=" + mainCurrencyPrice.ToString("#,#0.00") + " USD."); + + if ((completedTicks % 100) == 0) + { + log.DoLogInfo("BinanceFutures - Our magicbots are still at work, hang on... " + completedTicks + "/" + totalTicks + " ticks done..."); + } + } + } + + log.DoLogInfo("BinanceFutures - Initial market data created. Ready to go!"); + } + + } + } +} \ No newline at end of file diff --git a/Monitor/Pages/_get/DashboardTop.cshtml b/Monitor/Pages/_get/DashboardTop.cshtml index ade252f..e2c77f9 100644 --- a/Monitor/Pages/_get/DashboardTop.cshtml +++ b/Monitor/Pages/_get/DashboardTop.cshtml @@ -159,6 +159,9 @@ case "binanceus": ExchangeFee = 0.002; break; + case "binancefutures": + ExchangeFee = 0.002; + break; case "bittrex": ExchangeFee = 0.0025; break; diff --git a/PTMagic/Program.cs b/PTMagic/Program.cs index be0c197..c22e545 100644 --- a/PTMagic/Program.cs +++ b/PTMagic/Program.cs @@ -6,7 +6,7 @@ using Core.Helper; using Microsoft.Extensions.DependencyInjection; -[assembly: AssemblyVersion("3.0.0")] +[assembly: AssemblyVersion("2.4.0")] [assembly: AssemblyProduct("PT Magic")] namespace PTMagic diff --git a/_Development/CreateRelease.cmd b/_Development/CreateRelease.cmd index a98ba77..612a331 100644 --- a/_Development/CreateRelease.cmd +++ b/_Development/CreateRelease.cmd @@ -20,3 +20,5 @@ CD .\PTMagic DEL /F /S /Q _presets DEL /F /S /Q settings.* DEL /F /S /Q Monitor\appsettings.json + +pause