Merge pull request #87 from djbadders/develop

Fixes to marketdata in non UTC timezones + small tweaks

Core/Main/PTMagic.cs

@@ -604,6 +604,7 @@ namespace Core.Main
       this.Log.DoLogInfo("Detected a '" + e.ChangeType.ToString() + "' change in the following preset file: " + e.FullPath);
 
       // Reprocess now
+      this.EnforceSettingsReapply = true;
       PTMagicIntervalTimer_Elapsed(new object(), null);
 
       // Enable the file watcher again
@@ -964,7 +965,7 @@ namespace Core.Main
 
       FileInfo generalSettingsFile = new FileInfo(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings.general.json");
       FileInfo analyzerSettingsFile = new FileInfo(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings.analyzer.json");
-      if (generalSettingsFile.LastWriteTimeUtc > this.LastSettingFileCheck || analyzerSettingsFile.LastWriteTimeUtc > this.LastSettingFileCheck || EnforceSettingsReapply)
+      if (generalSettingsFile.LastWriteTimeUtc > this.LastSettingFileCheck || analyzerSettingsFile.LastWriteTimeUtc > this.LastSettingFileCheck)
       {
         Log.DoLogInfo("Detected configuration changes. Reloading settings...");
 
@@ -1202,7 +1203,6 @@ namespace Core.Main
       }
       else if (this.PTMagicConfiguration.GeneralSettings.Application.Exchange.Equals("Binance", StringComparison.InvariantCultureIgnoreCase))
       {
-
         // Get most recent market data from Binance
         this.ExchangeMarketList = Binance.GetMarketData(this.LastRuntimeSummary.MainMarket, this.MarketInfos, this.PTMagicConfiguration, this.Log);
       }
@@ -1382,14 +1382,7 @@ namespace Core.Main
       if (this.EnforceSettingsReapply || !activeSettingName.Equals(triggeredSetting.SettingName, StringComparison.InvariantCultureIgnoreCase))
       {
         // Check if we need to force a refresh of the settings
-        if (this.EnforceSettingsReapply)
+        this.Log.DoLogInfo("Setting '" + activeSettingName + "' currently active. Checking for flood protection...");
-        {
-          this.Log.DoLogInfo("Reapplying '" + activeSettingName + "' as the settings.analyzer.json or a preset file got changed.");
-        }
-        else
-        {
-          this.Log.DoLogInfo("Setting '" + activeSettingName + "' currently active. Checking for flood protection...");
-        }
 
         // If the setting we are about to activate is the default one, do not list matched triggers
         if (triggeredSetting.SettingName.Equals(this.DefaultSettingName, StringComparison.InvariantCultureIgnoreCase))
@@ -1405,6 +1398,10 @@ namespace Core.Main
           {
             this.Log.DoLogInfo("Switching global settings to '" + triggeredSetting.SettingName + "'...");
           }
+          else
+          {
+            this.Log.DoLogInfo("Applying '" + triggeredSetting.SettingName + "' as the settings.analyzer.json or a preset file got changed.");
+          }
 
           SettingsHandler.CompileProperties(this, triggeredSetting);
           this.GlobalSettingWritten = true;
@@ -1415,35 +1412,31 @@ namespace Core.Main
 
           // Build Telegram message
           string telegramMessage;
-          if (!EnforceSettingsReapply)
+          telegramMessage = this.PTMagicConfiguration.GeneralSettings.Application.InstanceName + ": Setting switched to '*" + SystemHelper.SplitCamelCase(triggeredSetting.SettingName) + "*'.";
+
+          if (matchedTriggers.Count > 0)
           {
-            telegramMessage = this.PTMagicConfiguration.GeneralSettings.Application.InstanceName + ": Setting switched to '*" + SystemHelper.SplitCamelCase(triggeredSetting.SettingName) + "*'.";
+            telegramMessage += "\n\n*Matching Triggers:*";
-
+            foreach (string triggerResult in matchedTriggers)
-            if (matchedTriggers.Count > 0)
             {
-              telegramMessage += "\n\n*Matching Triggers:*";
+              telegramMessage += "\n" + triggerResult;
-              foreach (string triggerResult in matchedTriggers)
-              {
-                telegramMessage += "\n" + triggerResult;
-              }
-            }
-
-            if (this.AverageMarketTrendChanges.Keys.Count > 0)
-            {
-              telegramMessage += "\n\n*Market Trends:*";
-              foreach (string key in this.AverageMarketTrendChanges.Keys)
-              {
-                telegramMessage += "\n" + key + ": " + this.AverageMarketTrendChanges[key].ToString("#,#0.00", new System.Globalization.CultureInfo("en-US")) + "%";
-              }
             }
           }
-          else
+
+          if (this.AverageMarketTrendChanges.Keys.Count > 0)
           {
-            telegramMessage = this.PTMagicConfiguration.GeneralSettings.Application.InstanceName + ": Reapplying settings '*" + SystemHelper.SplitCamelCase(triggeredSetting.SettingName) + "*'.";
+            telegramMessage += "\n\n*Market Trends:*";
+            foreach (string key in this.AverageMarketTrendChanges.Keys)
+            {
+              telegramMessage += "\n" + key + ": " + this.AverageMarketTrendChanges[key].ToString("#,#0.00", new System.Globalization.CultureInfo("en-US")) + "%";
+            }
           }
 
           // Send Telegram message
-          if (this.PTMagicConfiguration.GeneralSettings.Telegram.IsEnabled) TelegramHelper.SendMessage(this.PTMagicConfiguration.GeneralSettings.Telegram.BotToken, this.PTMagicConfiguration.GeneralSettings.Telegram.ChatId, telegramMessage, this.PTMagicConfiguration.GeneralSettings.Telegram.SilentMode, this.Log);
+          if (this.PTMagicConfiguration.GeneralSettings.Telegram.IsEnabled)
+          {
+            TelegramHelper.SendMessage(this.PTMagicConfiguration.GeneralSettings.Telegram.BotToken, this.PTMagicConfiguration.GeneralSettings.Telegram.ChatId, telegramMessage, this.PTMagicConfiguration.GeneralSettings.Telegram.SilentMode, this.Log);
+          }
 
           // Record last settings run
           this.LastSetting = activeSettingName;
@@ -2486,10 +2479,7 @@ namespace Core.Main
           }
         }
 
-        if (!this.LastRuntimeSummary.MarketSummary.ContainsKey(marketPair))
+        this.LastRuntimeSummary.MarketSummary.Add(marketPair, mpSummary);
-        {
-          this.LastRuntimeSummary.MarketSummary.Add(marketPair, mpSummary);
-        }
       }
 
       this.Log.DoLogInfo("Summary: Current single market properties saved.");

Core/MarketAnalyzer/BaseAnalyzer.cs

@@ -345,7 +345,17 @@ namespace Core.MarketAnalyzer
       return output;
     }
 
-    public static List<MarketTrendChange> GetMarketTrendChanges(string platform, string mainMarket, MarketTrend marketTrend, List<string> marketList, Dictionary<string, Market> recentMarkets, Dictionary<string, Market> trendMarkets, string sortBy, bool isGlobal, PTMagicConfiguration systemConfiguration, LogHelper log)
+    public static List<MarketTrendChange> GetMarketTrendChanges(
+      string platform, 
+      string mainMarket, 
+      MarketTrend marketTrend, 
+      List<string> marketList, 
+      Dictionary<string, Market> recentMarkets, 
+      Dictionary<string, Market> trendMarkets, 
+      string sortBy, 
+      bool isGlobal, 
+      PTMagicConfiguration systemConfiguration, 
+      LogHelper log)
     {
       List<MarketTrendChange> result = new List<MarketTrendChange>();
 
@@ -368,7 +378,6 @@ namespace Core.MarketAnalyzer
 
           if (platform.Equals("CoinMarketCap", StringComparison.InvariantCulture) && excludeMainCurrency)
           {
-
             // Check if this is the main currency (only for CoinMarketCap)
             if (recentMarketPair.Value.Symbol.Equals(mainMarket, StringComparison.InvariantCultureIgnoreCase))
             {
@@ -377,6 +386,7 @@ namespace Core.MarketAnalyzer
               continue;
             }
           }
+
           Market recentMarket = recentMarkets[recentMarketPair.Key];
 
           if (trendMarkets.ContainsKey(recentMarketPair.Key))

Core/MarketAnalyzer/Binance.cs

@@ -11,6 +11,7 @@ using Core.ProfitTrailer;
 using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Collections.Concurrent;
 
 namespace Core.MarketAnalyzer
 {
@@ -75,24 +76,24 @@ namespace Core.MarketAnalyzer
               float marketVolume = currencyTicker["volume"].ToObject<float>();
               if (marketName.EndsWith(mainMarket, StringComparison.InvariantCultureIgnoreCase))
               {
-                if(marketLastPrice > 0 && marketVolume > 0 )
+                if (marketLastPrice > 0 && marketVolume > 0)
                 {
 
-                // Set last values in case any error occurs
+                  // Set last values in case any error occurs
-                lastMarket = marketName;
+                  lastMarket = marketName;
-                lastTicker = currencyTicker;
+                  lastTicker = currencyTicker;
 
-                Market market = new Market();
+                  Market market = new Market();
-                market.Position = markets.Count + 1;
+                  market.Position = markets.Count + 1;
-                market.Name = marketName;
+                  market.Name = marketName;
-                market.Symbol = currencyTicker["symbol"].ToString();
+                  market.Symbol = currencyTicker["symbol"].ToString();
-                market.Price = SystemHelper.TextToDouble(currencyTicker["lastPrice"].ToString(), 0, "en-US");
+                  market.Price = SystemHelper.TextToDouble(currencyTicker["lastPrice"].ToString(), 0, "en-US");
-                market.Volume24h = SystemHelper.TextToDouble(currencyTicker["quoteVolume"].ToString(), 0, "en-US");
+                  market.Volume24h = SystemHelper.TextToDouble(currencyTicker["quoteVolume"].ToString(), 0, "en-US");
-                market.MainCurrencyPriceUSD = mainCurrencyPrice;
+                  market.MainCurrencyPriceUSD = mainCurrencyPrice;
 
-                markets.Add(market.Name, market);
+                  markets.Add(market.Name, market);
 
-                result.Add(market.Name);
+                  result.Add(market.Name);
                 }
                 else
                 {
@@ -108,7 +109,7 @@ namespace Core.MarketAnalyzer
 
             Binance.CheckForMarketDataRecreation(mainMarket, markets, systemConfiguration, log);
 
-            DateTime fileDateTime = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day, DateTime.UtcNow.Hour, DateTime.UtcNow.Minute, 0).ToUniversalTime();
+            DateTime fileDateTime = DateTime.UtcNow;
 
             FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(markets), fileDateTime, fileDateTime);
 
@@ -167,7 +168,7 @@ namespace Core.MarketAnalyzer
       log.DoLogInfo("Binance - Checking first seen dates for " + markets.Count + " markets. This may take a while...");
 
       int marketsChecked = 0;
-      
+
       foreach (string key in markets.Keys)
       {
         // Save market info
@@ -360,13 +361,17 @@ namespace Core.MarketAnalyzer
 
         // Get Ticks for all markets
         log.DoLogDebug("Binance - Getting ticks for '" + markets.Count + "' markets");
-        Dictionary<string, List<MarketTick>> marketTicks = new Dictionary<string, List<MarketTick>>();
+        ConcurrentDictionary<string, List<MarketTick>> marketTicks = new ConcurrentDictionary<string, List<MarketTick>>();
 
-        Parallel.ForEach( markets.Keys, 
+        Parallel.ForEach(markets.Keys,
                           new ParallelOptions { MaxDegreeOfParallelism = 5 },
                           (key) =>
         {
-          marketTicks.Add(key, Binance.GetMarketTicks(key, totalTicks, systemConfiguration, log));
+          if (!marketTicks.TryAdd(key, GetMarketTicks(key, totalTicks, systemConfiguration, log)))
+          {
+            // Failed to add ticks to dictionary
+            throw new Exception("Failed to add ticks for " + key + " to the memory dictionary, results may be incorrectly calculated!");
+          }
 
           if ((marketTicks.Count % 10) == 0)
           {
@@ -436,4 +441,4 @@ namespace Core.MarketAnalyzer
 
     }
   }
-}
+}

Core/MarketAnalyzer/Bittrex.cs

@@ -115,7 +115,7 @@ namespace Core.MarketAnalyzer
 
               Bittrex.CheckForMarketDataRecreation(mainMarket, markets, systemConfiguration, log);
 
-              DateTime fileDateTime = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day, DateTime.UtcNow.Hour, DateTime.UtcNow.Minute, 0).ToUniversalTime();
+              DateTime fileDateTime = DateTime.UtcNow;
 
               FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(markets), fileDateTime, fileDateTime);
 
@@ -235,11 +235,15 @@ namespace Core.MarketAnalyzer
         log.DoLogDebug("Bittrex - Getting ticks for '" + markets.Count + "' markets");
         Dictionary<string, List<MarketTick>> marketTicks = new Dictionary<string, List<MarketTick>>();
 
-        Parallel.ForEach( markets.Keys, 
+        Parallel.ForEach(markets.Keys,
                           new ParallelOptions { MaxDegreeOfParallelism = 5 },
                           (key) =>
         {
-          marketTicks.Add(key, Bittrex.GetMarketTicks(key, systemConfiguration, log));
+          if (!marketTicks.TryAdd(key, Bittrex.GetMarketTicks(key, systemConfiguration, log)))
+          {
+            // Failed to add ticks to dictionary
+            throw new Exception("Failed to add ticks for " + key + " to the memory dictionary, results may be incorrectly calculated!");
+          }
 
           if ((marketTicks.Count % 10) == 0)
           {

Core/MarketAnalyzer/CoinMarketCap.cs

@@ -53,11 +53,10 @@ namespace Core.MarketAnalyzer
 
             CoinMarketCap.CheckForMarketDataRecreation(markets, systemConfiguration, log);
 
-            DateTime fileDateTime = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day, DateTime.UtcNow.Hour, DateTime.UtcNow.Minute, 0).ToUniversalTime();
+            // Save the data
-
+            DateTime fileDateTime = DateTime.UtcNow;
             FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathCoinMarketCap + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(markets), fileDateTime, fileDateTime);
 
-
             log.DoLogInfo("CoinMarketCap - Market data saved.");
 
             FileHelper.CleanupFiles(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathCoinMarketCap + Path.DirectorySeparatorChar, systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours);
@@ -139,11 +138,10 @@ namespace Core.MarketAnalyzer
           markets24h.Add(markets[key].Name, market24h);
         }
 
-        DateTime fileDateTime = new DateTime(DateTime.UtcNow.ToLocalTime().AddHours(-24).Year, DateTime.UtcNow.ToLocalTime().AddHours(-24).Month, DateTime.UtcNow.ToLocalTime().AddHours(-24).Day, DateTime.UtcNow.ToLocalTime().AddHours(-24).Hour, DateTime.UtcNow.ToLocalTime().AddHours(-24).Minute, 0).ToUniversalTime();
+        // Save the 24hr market data
-
+        DateTime fileDateTime = DateTime.UtcNow.AddHours(-24);
         FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathCoinMarketCap + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(markets24h), fileDateTime, fileDateTime);
 
-
         log.DoLogInfo("CoinMarketCap - 24h market data rebuilt.");
       }
     }

Core/MarketAnalyzer/Poloniex.cs

@@ -99,7 +99,7 @@ namespace Core.MarketAnalyzer
 
             Poloniex.CheckForMarketDataRecreation(mainMarket, markets, systemConfiguration, log);
 
-            DateTime fileDateTime = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day, DateTime.UtcNow.Hour, DateTime.UtcNow.Minute, 0).ToUniversalTime();
+            DateTime fileDateTime = DateTime.UtcNow;
 
             FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(markets), fileDateTime, fileDateTime);
 
@@ -269,11 +269,15 @@ namespace Core.MarketAnalyzer
         log.DoLogDebug("Poloniex - Getting ticks for '" + markets.Count + "' markets");
         Dictionary<string, List<MarketTick>> marketTicks = new Dictionary<string, List<MarketTick>>();
 
-        Parallel.ForEach( markets.Keys, 
+        Parallel.ForEach(markets.Keys,
                           new ParallelOptions { MaxDegreeOfParallelism = 5 },
                           (key) =>
         {
-          marketTicks.Add(key, Poloniex.GetMarketTicks(key, systemConfiguration, log));
+          if (!marketTicks.TryAdd(key, Bittrex.GetMarketTicks(key, systemConfiguration, log)))
+          {
+            // Failed to add ticks to dictionary
+            throw new Exception("Failed to add ticks for " + key + " to the memory dictionary, results may be incorrectly calculated!");
+          }
 
           if ((marketTicks.Count % 10) == 0)
           {