1  using Exchanges.Instruments
  2  using .TimeTicks
  3  using .Misc: DFT, FUNDING_PERIOD
  4  using .Misc.TimeToLive
  5  using .Misc.Lang: @lget!, @debug_backtrace
  6  using .Instruments.Derivatives
  7  using .Python: PyDict
  8  using Processing: _fill_missing_candles
  9  using Processing.Data: Not, select!
 10  
 11  @doc """Retrieves all or a subset of funding data for a symbol from an exchange.
 12  
 13  $(TYPEDSIGNATURES)
 14  
 15  The `funding_data` function retrieves all funding data returned by an exchange `exc` for a symbol `sym`.
 16  """
 17  function funding_data(exc::Exchange, sym::AbstractString)
 18      try
 19          pyfetch(exc.fetchFundingRate, sym)
 20      catch
 21          @debug_backtrace
 22      end
 23  end
 24  funding_data(exc, a::Derivative, args...) = funding_data(exc, a.raw)
 25  funding_data(v, args...) = funding_data(exc, v, args...)
 26  
 27  @doc """Retrieves the funding rate for a symbol from an exchange.
 28  
 29  $(TYPEDSIGNATURES)
 30  
 31  The `funding_rate` function retrieves the funding rate for a symbol `s` from an exchange `exc`.
 32  """
 33  function funding_rate(exc::Exchange, s::AbstractString)
 34      id = exc.id
 35      @lget! FUNDING_RATE_CACHE (s, id) begin
 36          resp = if exc.has[:fetchFundingRates]
 37              rates = @lget! FUNDING_RATES_CACHE id pyfetch(exc.fetchFundingRates)
 38              rates[s]
 39          else
 40              pyfetch(exc.fetchFundingRate, s)
 41          end
 42          get(k) =
 43              let
 44                  v = (resp.get(k))
 45                  if Python.pyisTrue(pytype(v) == pybuiltins.str)
 46                      pytofloat(v)
 47                  else
 48                      pyconvert(Option{DFT}, v)
 49                  end
 50              end
 51          @something get("fundingRate") get("nextFundingRate") 0.00001
 52      end
 53  end
 54  funding_rate(exc, a::Derivative) = funding_rate(exc, a.raw)
 55  funding_rate(ai) = funding_rate(ai.exchange, ai.asset)
 56  
 57  const FUNDING_RATE_COLUMNS = (:timestamp, :pair, :rate)
 58  const FUNDING_RATE_COLS = [FUNDING_RATE_COLUMNS...]
 59  @doc """Parses a row of funding data from a Python object.
 60  
 61  $(TYPEDSIGNATURES)
 62  
 63  The `parse_funding_row` function takes a row of funding data `r` from a Python object and parses it into a format suitable for further processing or analysis.
 64  """
 65  function parse_funding_row(r::Py)
 66      pyconvert(Tuple{Int64,String,Float64}, (r["timestamp"], r["symbol"], r["fundingRate"]))
 67  end
 68  @doc """Extracts futures data from a Python object.
 69  
 70  $(TYPEDSIGNATURES)
 71  
 72  The `extract_futures_data` function takes futures data `data` from a Python object and extracts it into a format suitable for further processing or analysis.
 73  """
 74  function extract_futures_data(data::Py)
 75      ts, sym, rate = DateTime[], String[], Float64[]
 76      for r in data
 77          push!(ts, dt(pyconvert(Int64, r["timestamp"])))
 78          push!(sym, pyconvert(String, r["symbol"]))
 79          push!(rate, pyconvert(Float64, r["fundingRate"]))
 80      end
 81      DataFrame([ts, sym, rate], FUNDING_RATE_COLS)
 82  end
 83  
 84  @doc "Defines limit values for fetching futures data from exchanges."
 85  const futures_limits = IdDict(:binance => 1000)
 86  
 87  @doc """Fetches funding rate history from an exchange for a list of `Derivative` pairs.
 88  
 89  $(TYPEDSIGNATURES)
 90  
 91  The `funding_history` function fetches funding rate history from a given exchange `exc` for a list of `assets`. The `from` and `to` parameters define the date range for which to fetch the funding rate history. Additional parameters can be specified through the `params` dictionary. The function will wait for `sleep_t` seconds between each request to the exchange. The `limit` parameter can be used to limit the amount of data fetched. If `cleanup` is set to true, the function will perform a cleanup on the fetched data before returning it.
 92  """
 93  function funding_history(
 94      exc::Exchange,
 95      assets::Vector;
 96      from::DateType="",
 97      to::DateType="",
 98      params=Dict(),
 99      sleep_t=1,
100      limit=nothing,
101      cleanup=true,
102  )
103      from, to = from_to_dt(FUNDING_PERIOD, from, to)
104      from, to = _check_from_to(from, to)
105      ff =
106          (pair, since, limit; kwargs...) -> begin
107              try
108                  pyfetch(exc.py.fetchFundingRateHistory, pair; since, limit, params)
109              catch err
110                  # HACK: `since` is supposed to be the timestamp of the beginning of the
111                  # period to fetch. However if it considered invalid, use a negative value
112                  # representing the milliseconds that have passed since the start date.
113                  if occursin("Time Is Invalid", string(err))
114                      delta = -Int(timefloat(now() - dt(since)))
115                      pyfetch(
116                          exc.py.fetchFundingRateHistory, pair; since=delta, limit, params
117                      )
118                  else
119                      throw(err)
120                  end
121              end
122          end
123      if isnothing(limit)
124          limit = get(futures_limits, Symbol(exc.id), nothing)
125      end
126      ans = Dict(
127          begin
128              out = DataFrame(
129                  [DateTime[], String[], Float64[]], FUNDING_RATE_COLS; copycols=false
130              )
131              _fetch_loop(
132                  ff,
133                  exc,
134                  raw(a);
135                  from,
136                  to,
137                  sleep_t,
138                  converter=extract_futures_data,
139                  limit,
140                  out,
141              )
142              a => out
143          end for a in assets
144      )
145      if cleanup
146          # use a shorter timeframe to avoid overlapping
147          half_tf = TimeFrame(Millisecond(FUNDING_PERIOD) / 2)
148          f_tf = TimeFrame(Millisecond(FUNDING_PERIOD))
149          for k in keys(ans)
150              _cleanup_funding_history(ans[k], k, half_tf, f_tf)
151          end
152      end
153      return ans
154  end
155  
156  @doc """Cleans up fetched funding history data.
157  
158  $(TYPEDSIGNATURES)
159  
160  The `_cleanup_funding_history` function takes a DataFrame `df` of fetched funding history data for a `name` and performs cleanup operations on it. The `half_tf` and `f_tf` parameters are used in the cleanup process.
161  """
162  function _cleanup_funding_history(df, name, half_tf, f_tf)
163      # normalize timestamps
164      df.timestamp[:] = apply.(half_tf, df.timestamp)
165      unique!(df, :timestamp)
166      # resample to funding timestamp
167      resample(df, half_tf, f_tf)
168      # add close because of fill_missing_candles
169      df[!, :close] .= 0.0
170      buildf(ts, args...) = (; timestamp=ts, pair=string(name), rate=0.0001, close=NaN)
171      _fill_missing_candles(
172          df,
173          FUNDING_PERIOD;
174          strategy=:custom,
175          inplace=true,
176          def_strategy=buildf,
177          def_type=NamedTuple{
178              (:timestamp, :pair, :rate, :close),Tuple{DateTime,String,DFT,DFT}
179          },
180      )
181      # remove close after fills
182      select!(df, Not(:close))
183  end
184  
185  @doc "Defines the time-to-live (TTL) for a funding rate as 5 seconds."
186  const FUNDING_RATE_TTL = Ref(Second(5))
187  @doc "Initializes a safe TTL cache for storing funding rates with a specified TTL."
188  const FUNDING_RATE_CACHE = safettl(Tuple{String,Symbol}, DFT, FUNDING_RATE_TTL[])
189  @doc "Initializes a safe TTL cache for storing multiple funding rates with a specified TTL."
190  const FUNDING_RATES_CACHE = safettl(Symbol, Py, FUNDING_RATE_TTL[])
191  assetkey(ai) = (ai.raw, ai.exchange.id)
192  
193  export funding_history, funding_rate