1  #!/usr/bin/env python3
  2  
  3  
  4  # sum(shh_messages_total) by (chat)
  5  
  6  
  7  def openPrometheusURL(query, timestep):
  8      from time import time
  9      from urllib.request import urlopen
 10  
 11      # TODO: could escape it, but could also just avoid queries which need escaping
 12  
 13      cur_time = time()
 14      return urlopen(
 15          "http://master-01.do-ams3.metrics.hq.tinc:9090/api/v1/query_range?query=%s&end=%d&start=%d&step=%s"
 16          % (query, time(), time() - 3600 * 24 * 60, timestep)
 17      ).read()
 18  
 19  
 20  def parse_data(raw_json):
 21      from json import loads
 22  
 23      j = loads(raw_json)
 24      assert j["status"] == "success"
 25      assert j["data"]["resultType"] == "matrix"
 26      return j["data"]["result"][0]["values"]
 27  
 28  
 29  def getData(pj):
 30      from datetime import datetime
 31  
 32      return list(
 33          zip(
 34              *[
 35                  (datetime.fromtimestamp(z[0]), float(z[1]))
 36                  for z in sorted(pj, key=lambda _: _[0])
 37              ]
 38          )
 39      )
 40  
 41  
 42  def plotData(filename, ylabel, timespan, x, y):
 43      # TODO: factor out as common routine
 44      # allow this to run on a headless server
 45      import matplotlib
 46  
 47      matplotlib.use("Agg")
 48  
 49      import matplotlib.pyplot as plt
 50      import matplotlib.dates as mdates
 51  
 52      assert len(x) == len(y)
 53  
 54      # https://stackoverflow.com/questions/9627686/plotting-dates-on-the-x-axis-with-pythons-matplotlib#9627970
 55      plt.clf()
 56      plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d"))
 57      plt.gca().xaxis.set_major_locator(mdates.AutoDateLocator())
 58      plt.xlabel("Date")
 59  
 60      plt.ylabel(ylabel)
 61  
 62      plt.title("Status.im %s %s (Prometheus)" % (timespan, ylabel))
 63  
 64      plt.plot(x, y)
 65      plt.gcf().autofmt_xdate()
 66      plt.savefig(filename, dpi=150)
 67  
 68  
 69  def distinct_users():
 70      from collections import Counter
 71      from datetime import datetime
 72      from json import loads
 73  
 74      j = loads(
 75          openPrometheusURL(
 76              "sum(rate(shh_messages_total[1d])*3600)%20by%20(source)", "1d"
 77          )
 78      )
 79      assert j["status"] == "success"
 80      assert j["data"]["resultType"] == "matrix"
 81  
 82      distinct_user_count = Counter()
 83      for metric_values in j["data"]["result"]:
 84          source = metric_values["metric"].get("source", "")
 85          values = metric_values["values"]
 86          for epoch_time, hourly_rate in values:
 87              if float(hourly_rate) > 0:
 88                  distinct_user_count[datetime.fromtimestamp(epoch_time)] += 1
 89  
 90      # TODO: refactor, etc
 91      return list(zip(*sorted(distinct_user_count.items())))
 92  
 93  
 94  def main():
 95      from os.path import join
 96      from sys import argv
 97  
 98      getPath = lambda n: join(argv[1], n)
 99  
100      plotData(
101          getPath("whisper_distinct_users_daily.png"),
102          "Whisper Users (Public Channels)",
103          "Daily",
104          *distinct_users()
105      )
106  
107      plotData(
108          getPath("whisper_messages_hourly.png"),
109          "Whisper Messages (Public Channels)",
110          "Hourly",
111          *getData(
112              parse_data(
113                  openPrometheusURL("sum(rate(shh_messages_total[1h]))*3600", "1h")
114              )
115          )
116      )
117  
118      plotData(
119          getPath("whisper_messages_daily.png"),
120          "Whisper Messages (Public Channels)",
121          "Daily",
122          *getData(
123              parse_data(
124                  openPrometheusURL("sum(rate(shh_messages_total[1d]))*3600*24", "1d")
125              )
126          )
127      )
128  
129      plotData(
130          getPath("whisper_messages_weekly.png"),
131          "Whisper Messages (Public Channels)",
132          "Weekly",
133          *getData(
134              parse_data(
135                  openPrometheusURL("sum(rate(shh_messages_total[1w]))*3600*24*7", "1w")
136              )
137          )
138      )
139  
140  
141  main()