Gemini ticker web socket and symbol metadata fix

Author: jjxtra

src/ExchangeSharp/API/Exchanges/Gemini/ExchangeGeminiAPI.cs

@@ -11,11 +11,13 @@ The above copyright notice and this permission notice shall be included in all c
 */
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
 using System.Text;
 using System.Threading.Tasks;
+using System.Web;
 
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -30,6 +32,7 @@ public ExchangeGeminiAPI()
         {
             MarketSymbolIsUppercase = false;
             MarketSymbolSeparator = string.Empty;
+			RateLimit = new RateGate(1, TimeSpan.FromSeconds(0.5));
         }
 
         private async Task<ExchangeVolume> ParseVolumeAsync(JToken token, string symbol)
@@ -93,63 +96,38 @@ protected override async Task<IEnumerable<string>> OnGetMarketSymbolsAsync()
 
 		protected internal override async Task<IEnumerable<ExchangeMarket>> OnGetMarketSymbolsMetadataAsync()
 		{
-			List<ExchangeMarket> hardcodedSymbols = new List<ExchangeMarket>()
+			Logger.Warn("Fetching gemini symbol metadata, this may take a minute...");
+
+			string[] symbols = (await GetMarketSymbolsAsync()).ToArray();
+			List<ExchangeMarket> markets = new List<ExchangeMarket>();
+			List<Task> tasks = new List<Task>();
+			foreach (string symbol in symbols)
 			{
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "btcusd", BaseCurrency = "BTC", QuoteCurrency = "USD",
-					MinTradeSize = 0.00001M, QuantityStepSize = 0.00000001M, PriceStepSize = 0.01M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "ethusd", BaseCurrency = "ETH", QuoteCurrency = "USD",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.01M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "ethbtc", BaseCurrency = "ETH", QuoteCurrency = "BTC",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.00001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "zecusd", BaseCurrency = "ZEC", QuoteCurrency = "USD",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.01M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "zecbtc", BaseCurrency = "ZEC", QuoteCurrency = "BTC",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.00001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "zeceth", BaseCurrency = "ZEC", QuoteCurrency = "ETH",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.0001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "zecbch", BaseCurrency = "ZEC", QuoteCurrency = "BCH",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.0001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "zecltc", BaseCurrency = "ZEC", QuoteCurrency = "LTC",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "bchusd", BaseCurrency = "BCH", QuoteCurrency = "USD",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.01M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "bchbtc", BaseCurrency = "BCH", QuoteCurrency = "BTC",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.00001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "bcheth", BaseCurrency = "BCH", QuoteCurrency = "ETH",
-					MinTradeSize = 0.001M, QuantityStepSize = 0.000001M, PriceStepSize = 0.0001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "ltcusd", BaseCurrency = "LTC", QuoteCurrency = "USD",
-					MinTradeSize = 0.01M, QuantityStepSize = 0.00001M, PriceStepSize = 0.01M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "ltcbtc", BaseCurrency = "LTC", QuoteCurrency = "BTC",
-					MinTradeSize = 0.01M, QuantityStepSize = 0.00001M, PriceStepSize = 0.00001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "ltceth", BaseCurrency = "LTC", QuoteCurrency = "ETH",
-					MinTradeSize = 0.01M, QuantityStepSize = 0.00001M, PriceStepSize = 0.0001M},
-				new ExchangeMarket() { IsActive = true,
-					MarketSymbol = "ltcbch", BaseCurrency = "LTC", QuoteCurrency = "BCH",
-					MinTradeSize = 0.01M, QuantityStepSize = 0.00001M, PriceStepSize = 0.0001M},
-			};
-			// + check to make sure no symbols are missing
-			var apiSymbols = await GetMarketSymbolsAsync();
-			foreach (var apiSymbol in apiSymbols)
-				if (!hardcodedSymbols.Select(m => m.MarketSymbol).Contains(apiSymbol))
-					throw new Exception("hardcoded symbols out of date, please send a PR on GitHub to update.");
-			foreach (var hardcodedSymbol in hardcodedSymbols)
-				if (!apiSymbols.Contains(hardcodedSymbol.MarketSymbol))
-					throw new Exception("hardcoded symbols out of date, please send a PR on GitHub to update.");
-			return hardcodedSymbols;
+				tasks.Add(Task.Run(async () =>
+				{
+					JToken token = await MakeJsonRequestAsync<JToken>("/symbols/details/" + HttpUtility.UrlEncode(symbol));
+
+					// {"symbol":"BTCUSD","base_currency":"BTC","quote_currency":"USD","tick_size":1E-8,"quote_increment":0.01,"min_order_size":"0.00001","status":"open"}
+					lock (markets)
+					{
+						markets.Add(new ExchangeMarket
+						{
+							BaseCurrency = token["base_currency"].ToStringInvariant(),
+							IsActive = token["status"].ToStringInvariant().Equals("open", StringComparison.OrdinalIgnoreCase),
+							MarketSymbol = token["symbol"].ToStringInvariant(),
+							MinTradeSize = token["min_order_Size"].ConvertInvariant<decimal>(),
+							QuantityStepSize = token["tick_size"].ConvertInvariant<decimal>(),
+							QuoteCurrency = token["quote_currency"].ToStringInvariant(),
+							PriceStepSize = token["quote_increment"].ConvertInvariant<decimal>()
+						});
+					}
+				}));
+			}
+			await Task.WhenAll(tasks);
+
+			Logger.Warn("Gemini symbol metadata fetched and cached for several hours.");
+
+			return markets;
 		}
 
 		protected override async Task<ExchangeTicker> OnGetTickerAsync(string marketSymbol)
@@ -284,6 +262,98 @@ protected override async Task OnCancelOrderAsync(string orderId, string marketSy
             await MakeJsonRequestAsync<JToken>("/order/cancel", null, new Dictionary<string, object>{ { "nonce", nonce }, { "order_id", orderId } });
         }
 
+		protected override async Task<IWebSocket> OnGetTickersWebSocketAsync(Action<IReadOnlyCollection<KeyValuePair<string, ExchangeTicker>>> tickers, params string[] marketSymbols)
+		{
+			if (marketSymbols == null || marketSymbols.Length == 0)
+			{
+				marketSymbols = (await GetMarketSymbolsAsync()).ToArray();
+			}
+			ConcurrentDictionary<string, decimal> volumeDict = new ConcurrentDictionary<string, decimal>();
+			ConcurrentDictionary<string, ExchangeTicker> tickerDict = new ConcurrentDictionary<string, ExchangeTicker>();
+			return await ConnectWebSocketAsync(null, messageCallback: async (_socket, msg) =>
+			{
+				JToken token = JToken.Parse(msg.ToStringFromUTF8());
+				if (token["result"].ToStringInvariant() == "error")
+				{
+					// {{  "result": "error",  "reason": "InvalidJson"}}
+					Logger.Info(token["reason"].ToStringInvariant());
+				}
+				else if (token["type"].ToStringInvariant() == "candles_1d_updates")
+				{
+					JToken changesToken = token["changes"];
+					string marketSymbol = token["symbol"].ToStringInvariant();
+					if (changesToken != null)
+					{
+						if (changesToken.FirstOrDefault() is JArray candleArray)
+						{
+							decimal volume = candleArray[5].ConvertInvariant<decimal>();
+							volumeDict[marketSymbol] = volume;
+						}
+					}
+				}
+				else if (token["type"].ToStringInvariant() == "l2_updates")
+				{
+					// fetch the active ticker metadata for this symbol
+					string marketSymbol = token["symbol"].ToStringInvariant();
+					ExchangeTicker ticker = tickerDict.GetOrAdd(marketSymbol, (_marketSymbol) =>
+					{
+						(string baseCurrency, string quoteCurrency) = ExchangeMarketSymbolToCurrenciesAsync(marketSymbol).Sync();
+						return new ExchangeTicker
+						{
+							MarketSymbol = _marketSymbol,
+							Volume = new ExchangeVolume
+							{
+								BaseCurrency = baseCurrency,
+								QuoteCurrency = quoteCurrency
+							}
+						};
+					});
+
+					// fetch the last bid/ask/last prices
+					if (token["changes"] is JArray changesToken)
+					{
+						if (changesToken.FirstOrDefault(t => t[0].ToStringInvariant().Equals("buy", StringComparison.OrdinalIgnoreCase)) is JArray buyToken)
+						{
+							decimal bidPrice = buyToken[1].ConvertInvariant<decimal>();
+							ticker.Bid = bidPrice;
+						}
+
+						if (changesToken.FirstOrDefault(t => t[0].ToStringInvariant().Equals("sell", StringComparison.OrdinalIgnoreCase)) is JArray sellToken)
+						{
+							decimal askPrice = sellToken[1].ConvertInvariant<decimal>();
+							ticker.Ask = askPrice;
+						}
+
+						if (token["trades"] is JArray tradesToken)
+						{
+							JToken lastTrade = tradesToken.FirstOrDefault();
+							if (lastTrade != null)
+							{
+								decimal lastTradePrice = lastTrade["price"].ConvertInvariant<decimal>();
+								ticker.Last = lastTradePrice;
+							}
+						}
+
+						// see if we have volume yet
+						if (volumeDict.TryGetValue(marketSymbol, out decimal tickerVolume))
+						{
+							ticker.Volume.BaseCurrencyVolume = tickerVolume;
+							ticker.Volume.QuoteCurrencyVolume = tickerVolume * ticker.Last;
+							var kv = new KeyValuePair<string, ExchangeTicker>(marketSymbol, ticker);
+							tickers(new KeyValuePair<string, ExchangeTicker>[] { kv });
+						}
+					}
+				}
+			}, connectCallback: async (_socket) =>
+			{
+				await _socket.SendMessageAsync(new
+				{
+					type = "subscribe",
+					subscriptions = new[] { new { name = "candles_1d", symbols = marketSymbols }, new { name = "l2", symbols = marketSymbols } }
+				});
+			});
+		}
+
 		protected override async Task<IWebSocket> OnGetTradesWebSocketAsync(Func<KeyValuePair<string, ExchangeTrade>, Task> callback, params string[] marketSymbols)
 		{
 			//{
@@ -358,6 +428,7 @@ protected override async Task<IWebSocket> OnGetTradesWebSocketAsync(Func<KeyValu
 			{
 				marketSymbols = (await GetMarketSymbolsAsync()).ToArray();
 			}
+
 			return await ConnectWebSocketAsync(BaseUrlWebSocket, messageCallback: async (_socket, msg) =>
 			{
 				JToken token = JToken.Parse(msg.ToStringFromUTF8());
@@ -371,15 +442,15 @@ protected override async Task<IWebSocket> OnGetTradesWebSocketAsync(Func<KeyValu
 					var tradesToken = token["trades"];
 					if (tradesToken != null) foreach (var tradeToken in tradesToken)
 					{
-						var trade = parseTrade(tradeToken);
+						var trade = ParseWebSocketTrade(tradeToken);
 						trade.Flags |= ExchangeTradeFlags.IsFromSnapshot;
 						await callback(new KeyValuePair<string, ExchangeTrade>(marketSymbol, trade));
 					}
 				}
 				else if (token["type"].ToStringInvariant() == "trade")
 				{
 					string marketSymbol = token["symbol"].ToStringInvariant();
-					var trade = parseTrade(token);
+					var trade = ParseWebSocketTrade(token);
 					await callback(new KeyValuePair<string, ExchangeTrade>(marketSymbol, trade));
 				}
 			}, connectCallback: async (_socket) =>
@@ -388,11 +459,12 @@ protected override async Task<IWebSocket> OnGetTradesWebSocketAsync(Func<KeyValu
 				await _socket.SendMessageAsync(new {
 						type = "subscribe", subscriptions = new[] { new { name = "l2", symbols = marketSymbols } } });
 			});
-			ExchangeTrade parseTrade(JToken token) => token.ParseTrade(
-							amountKey: "quantity", priceKey: "price",
-							typeKey: "side", timestampKey: "timestamp",
-							TimestampType.UnixMilliseconds, idKey: "event_id");
 		}
+
+		private static ExchangeTrade ParseWebSocketTrade(JToken token) => token.ParseTrade(
+			amountKey: "quantity", priceKey: "price",
+			typeKey: "side", timestampKey: "timestamp",
+			TimestampType.UnixMilliseconds, idKey: "event_id");
 	}
 
     public partial class ExchangeName { public const string Gemini = "Gemini"; }

src/ExchangeSharp/API/Exchanges/Kraken/ExchangeKrakenAPI.cs

@@ -57,7 +57,7 @@ static ExchangeKrakenAPI()
         /// <returns>Task</returns>
         private async Task PopulateLookupTables()
         {
-            await Cache.Get<object>(nameof(PopulateLookupTables), async () =>
+            await Cache.GetOrCreate<object>(nameof(PopulateLookupTables), async () =>
             {
                 IReadOnlyDictionary<string, ExchangeCurrency> currencies = await GetCurrenciesAsync();
                 ExchangeMarket[] markets = (await GetMarketSymbolsMetadataAsync())?.ToArray();

src/ExchangeSharp/API/Exchanges/ZBcom/ExchangeZBcomAPI.cs

@@ -122,7 +122,7 @@ protected override async Task<IEnumerable<KeyValuePair<string, ExchangeTicker>>>
 
             var data = await MakeRequestZBcomAsync(null, "/allTicker", BaseUrl);
             List<KeyValuePair<string, ExchangeTicker>> tickers = new List<KeyValuePair<string, ExchangeTicker>>();
-            var symbolLookup = await Cache.Get<Dictionary<string, string>>(nameof(GetMarketSymbolsAsync) + "_Set", async () =>
+            var symbolLookup = await Cache.GetOrCreate<Dictionary<string, string>>(nameof(GetMarketSymbolsAsync) + "_Set", async () =>
             {
                 // create lookup dictionary of symbol string without separator to symbol string with separator
                 IEnumerable<string> symbols = await GetMarketSymbolsAsync();

src/ExchangeSharp/API/Exchanges/_Base/ExchangeAPI.cs

@@ -252,7 +252,7 @@ public ExchangeAPI()
             {
                 MethodCachePolicy.Add(nameof(GetCurrenciesAsync), TimeSpan.FromHours(1.0));
                 MethodCachePolicy.Add(nameof(GetMarketSymbolsAsync), TimeSpan.FromHours(1.0));
-                MethodCachePolicy.Add(nameof(GetMarketSymbolsMetadataAsync), TimeSpan.FromHours(4.0));
+                MethodCachePolicy.Add(nameof(GetMarketSymbolsMetadataAsync), TimeSpan.FromHours(6.0));
                 MethodCachePolicy.Add(nameof(GetTickerAsync), TimeSpan.FromSeconds(10.0));
                 MethodCachePolicy.Add(nameof(GetTickersAsync), TimeSpan.FromSeconds(10.0));
                 MethodCachePolicy.Add(nameof(GetOrderBookAsync), TimeSpan.FromSeconds(10.0));

src/ExchangeSharp/API/Exchanges/_Base/ExchangeAPIExtensions.cs

@@ -203,7 +203,7 @@ async Task innerCallback(ExchangeOrderBook newOrderBook)
         public static async Task<Dictionary<string, ExchangeMarket>> GetExchangeMarketDictionaryFromCacheAsync(this ExchangeAPI api)
         {
             await new SynchronizationContextRemover();
-            CachedItem<Dictionary<string, ExchangeMarket>> cacheResult = await api.Cache.Get<Dictionary<string, ExchangeMarket>>(nameof(GetExchangeMarketDictionaryFromCacheAsync), async () =>
+            CachedItem<Dictionary<string, ExchangeMarket>> cacheResult = await api.Cache.GetOrCreate<Dictionary<string, ExchangeMarket>>(nameof(GetExchangeMarketDictionaryFromCacheAsync), async () =>
             {
                 try
                 {

src/ExchangeSharp/Model/ExchangeTicker.cs

@@ -1,4 +1,4 @@
-/*
+/*
 MIT LICENSE
 
 Copyright 2017 Digital Ruby, LLC - http://www.digitalruby.com
@@ -62,7 +62,7 @@ public sealed class ExchangeTicker
         /// <returns>String</returns>
         public override string ToString()
         {
-            return string.Format("Bid: {0}, Ask: {1}, Last: {2}", Bid, Ask, Last);
+            return string.Format("Bid: {0}, Ask: {1}, Last: {2}, Vol: {3}", Bid, Ask, Last, Volume);
         }
 
         /// <summary>
@@ -125,11 +125,17 @@ public sealed class ExchangeVolume
         /// </summary>
         public decimal BaseCurrencyVolume { get; set; }
 
-        /// <summary>
-        /// Write to a binary writer
-        /// </summary>
-        /// <param name="writer">Binary writer</param>
-        public void ToBinary(BinaryWriter writer)
+		/// <inheritdoc />
+		public override string ToString()
+		{
+			return $"{BaseCurrencyVolume:0.0000}/{QuoteCurrencyVolume:0.0000}";
+		}
+
+		/// <summary>
+		/// Write to a binary writer
+		/// </summary>
+		/// <param name="writer">Binary writer</param>
+		public void ToBinary(BinaryWriter writer)
         {
             writer.Write(Timestamp.ToUniversalTime().Ticks);
             writer.Write(QuoteCurrency);

src/ExchangeSharp/Utility/CryptoUtility.cs

@@ -1401,7 +1401,7 @@ public static async Task<T> CacheMethod<T>(this ICache cache, Dictionary<string,
             }
             if (methodCachePolicy.TryGetValue(methodName, out TimeSpan cacheTime))
             {
-                return (await cache.Get<T>(cacheKey, async () =>
+                return (await cache.GetOrCreate<T>(cacheKey, async () =>
                 {
                     T innerResult = await method();
                     return new CachedItem<T>(innerResult, CryptoUtility.UtcNow.Add(cacheTime));

src/ExchangeSharp/Utility/MemoryCache.cs

@@ -1,4 +1,4 @@
-/*
+/*
 MIT LICENSE
 
 Copyright 2017 Digital Ruby, LLC - http://www.digitalruby.com
@@ -64,8 +64,8 @@ public interface ICache : IDisposable
         /// <typeparam name="T">Type to read</typeparam>
         /// <param name="key">Key</param>
         /// <param name="value">Value</param>
-        /// <param name="notFound">Create T if not found, null to not do this. Item1 = value, Item2 = expiration.</param>
-        Task<CachedItem<T>> Get<T>(string key, Func<Task<CachedItem<T>>> notFound) where T : class;
+        /// <param name="factory">Create T if not found, null to not do this. Item1 = value, Item2 = expiration.</param>
+        Task<CachedItem<T>> GetOrCreate<T>(string key, Func<Task<CachedItem<T>>> factory) where T : class;
 
         /// <summary>
         /// Remove a key from the cache immediately
@@ -176,7 +176,7 @@ public void Dispose()
         /// <param name="key">Key</param>
         /// <param name="value">Value</param>
         /// <param name="notFound">Create T if not found, null to not do this. Item1 = value, Item2 = expiration.</param>
-        public async Task<CachedItem<T>> Get<T>(string key, Func<Task<CachedItem<T>>> notFound) where T : class
+        public async Task<CachedItem<T>> GetOrCreate<T>(string key, Func<Task<CachedItem<T>>> notFound) where T : class
         {
             CachedItem<T> newItem = default;
             cacheTimerLock.EnterReadLock();