1  /**
  2   * Created by ls on 9/7/2018.
  3   *
  4   * This `network.ts` is the NON-MEDIA API arm of network fetch requests.
  5   * It is built on `Network` object and provides standard functionality, such as:
  6   *     1. Parsing the body into specific format.
  7   *     2. Adding timing metrics onto blob.
  8   *
  9   * This should *only* be used for objects that should have timing metrics, i.e. requests to Non-MediaAPI endpoints
 10   * that will ultimately render some whole page. Otherwise, use `objectGraph.network.fetch` directly.
 11   *
 12   * @see `src/media/network.ts` for fetching from Media API endpoints
 13   */
 14  
 15  import { FetchRequest, FetchResponse, HTTPTimingMetrics } from "@jet/environment/types/globals/net";
 16  import { FetchTimingMetrics, MetricsFields } from "@jet/environment/types/metrics";
 17  import { FetchTimingMetricsBuilder } from "@jet/environment/metrics";
 18  import { isSome, Opt } from "@jet/environment/types/optional";
 19  import * as serverData from "./models/server-data";
 20  import { ParsedNetworkResponse } from "./models/data-structure";
 21  import { Request } from "./models/request";
 22  import * as urls from "./models/urls";
 23  import * as urlBuilder from "./url-builder";
 24  import { MediaConfigurationType, MediaTokenService } from "./models/mediapi-configuration";
 25  import { HTTPMethod, HTTPCachePolicy, HTTPSigningStyle, HTTPHeaders } from "@jet/environment";
 26  
 27  /** @public */
 28  // eslint-disable-next-line @typescript-eslint/no-namespace
 29  export namespace ResponseMetadata {
 30      export const requestedUrl = "_jet-internal:metricsHelpers_requestedUrl";
 31  
 32      /**
 33       * Symbol used to place timing metrics values onto fetch responses
 34       * without interfering with the data returned by the server.
 35       */
 36      export const timingValues = "_jet-internal:metricsHelpers_timingValues";
 37  
 38      /**
 39       * Key used to access the page information gathered from a response's headers
 40       */
 41      export const pageInformation = "_jet-internal:metricsHelpers_pageInformation";
 42  
 43      /**
 44       * Key used to access the content max-age gathered from a response's headers.
 45       */
 46      export const contentMaxAge = "_jet-internal:responseMetadata_contentMaxAge";
 47  }
 48  
 49  /**
 50   * Module's private fetch implementation built off `net` global.
 51   *
 52   * @param {FetchRequest} request describes fetch request.
 53   * @param {(value: string) => Type} parser Some function parsing response body `string` into specific type.
 54   * @returns {Promise<Type>} Promise resolving to specific object.
 55   * @throws {Error} Throws error if status code of request is not 200.
 56   *
 57   * @note Similar to `fetchWithToken` in `media` module, but excludes media token specific functionality.
 58   * Top level data fetches to endpoints that don't do redirects, and can benefit from metrics should
 59   * call methods that build off of this instead of calling `objectGraph.network.fetch(...)` directly.
 60   */
 61  export async function fetch<Type>(
 62      configuration: MediaConfigurationType,
 63      request: FetchRequest,
 64      parser: (value: Opt<string>) => Type,
 65  ): Promise<Type & ParsedNetworkResponse> {
 66      const response = await configuration.network.fetch(request);
 67      if (!response.ok) {
 68          throw Error(`Bad Status code ${response.status} for ${request.url}`);
 69      }
 70      const parseStartTime = Date.now();
 71      const result = parser(response.body) as Type & ParsedNetworkResponse;
 72      const parseEndTime = Date.now();
 73  
 74      // Build full network timing metrics.
 75      const completeTimingMetrics = networkTimingMetricsWithParseTime(response.metrics, parseStartTime, parseEndTime);
 76      if (serverData.isDefinedNonNull(completeTimingMetrics)) {
 77          result[ResponseMetadata.timingValues] = completeTimingMetrics;
 78      }
 79      result[ResponseMetadata.requestedUrl] = request.url.toString();
 80      return result;
 81  }
 82  
 83  /**
 84   * Fetch from an endpoint with JSON response body.
 85   *
 86   * @param {FetchRequest} request to fetch from endpoint with JSON response..
 87   * @returns {Promise<Type>} Promise resolving to body of response parsed as `Type`.
 88   * @throws {Error} Throws error if status code of request is not 200.
 89   */
 90  export async function fetchJSON<Type>(configuration: MediaConfigurationType, request: FetchRequest): Promise<Type> {
 91      return await fetch(configuration, request, (body) => {
 92          if (isSome(body)) {
 93              return JSON.parse(body) as Type;
 94          } else {
 95              // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
 96              return {} as Type;
 97          }
 98      });
 99  }
100  
101  /**
102   * Fetch from an endpoint with XML response body.
103   *
104   * @param {FetchRequest} request to fetch from endpoint with XML response.
105   * @returns {Promise<Type>} Promise resolving to body of response parsed as `Type`.
106   * @throws {Error} Throws error if status code of request is not 200.
107   */
108  export async function fetchPlist<Type>(configuration: MediaConfigurationType, request: FetchRequest): Promise<Type> {
109      return await fetch(configuration, request, (body) => {
110          if (isSome(body)) {
111              return configuration.plist.parse(body) as Type;
112          } else {
113              throw new Error(`Could not fetch Plist, response body was not defined for ${request.url}`);
114          }
115      });
116  }
117  
118  /**
119   * With network requests now being created and parsed in JS, different timing metrics are measured in both Native and JS.
120   * This function populates the missing values from `HTTPTimingMetrics`'s native counterpart, `JSNetworkPerformanceMetrics`.
121   *
122   * @param {HTTPTimingMetrics[] | null} responseMetrics Array of response metrics provided by native.
123   * @param {number} parseStartTime Time at which response body string parse began in JS.
124   * @param {number} parseEndTime Time at which response body string parse ended in JS.
125   * @returns {HTTPTimingMetrics | null} Fully populated timing metrics, or `null` if native response provided no metrics events to build off of.
126   */
127  function networkTimingMetricsWithParseTime(
128      responseMetrics: HTTPTimingMetrics[] | null,
129      parseStartTime: number,
130      parseEndTime: number,
131  ): FetchTimingMetrics | null {
132      // No metrics events to build from.
133      if (serverData.isNull(responseMetrics) || responseMetrics.length === 0) {
134          return null;
135      }
136  
137      // Append parse times to first partial timing metrics from native.
138      const firstPartialTimingMetrics: FetchTimingMetrics = {
139          ...responseMetrics[0],
140          parseStartTime: parseStartTime,
141          parseEndTime: parseEndTime,
142      };
143      // Timing metrics with all properties populated.
144      return firstPartialTimingMetrics;
145  }
146  
147  export type FetchOptions = {
148      headers?: { [key: string]: string };
149      method?: "GET" | "HEAD" | "POST" | "PUT" | "DELETE" | "CONNECT" | "OPTIONS" | "TRACE" | "PATCH";
150      requestBodyString?: string;
151      timeout?: number; // in seconds. Check for feature 'supportsRequestTimeoutOption'.
152      /// When true the fetch wont throw if we dont get any data back for given request.
153      allowEmptyDataResponse?: boolean;
154      excludeIdentifierHeadersForAccount?: boolean; // Defaults to false
155      alwaysIncludeAuthKitHeaders?: boolean; // Defaults to true
156      alwaysIncludeMMeClientInfoAndDeviceHeaders?: boolean; // Defaults to true
157  };
158  
159  /**
160   * Implements the MAPI fetch, building URL from MAPI Request and opaquely managing initial token request and refreshes.
161   *
162   * @param {MediaConfigurationType} configuration Base media API configuration.
163   * @param {Request} request MAPI Request to fetch with.
164   * @param {FetchOptions} [options] FetchOptions for the MAPI request.
165   * @returns {Promise<Type>} Promise resolving to some type for given MAPI request.
166   */
167  export async function fetchData<Type>(
168      configuration: MediaConfigurationType,
169      mediaToken: MediaTokenService,
170      request: Request,
171      options?: FetchOptions,
172  ): Promise<Type & ParsedNetworkResponse> {
173      const url = urlBuilder.buildURLFromRequest(configuration, request).toString();
174      const startTime = Date.now();
175      const token = await mediaToken.refreshToken();
176      const response = await fetchWithToken<Type>(
177          configuration,
178          mediaToken,
179          url,
180          token,
181          options,
182          false,
183          configuration.fetchTimingMetricsBuilder,
184      );
185      const endTime = Date.now();
186      if (request.canonicalUrl) {
187          response[ResponseMetadata.requestedUrl] = request.canonicalUrl;
188      }
189      const roundTripTimeIncludingWaiting = endTime - startTime;
190      if (roundTripTimeIncludingWaiting > 500) {
191          console.warn("Fetch took too long (" + roundTripTimeIncludingWaiting.toString() + "ms) " + url);
192      }
193      return response;
194  }
195  
196  export function redirectParametersInUrl(configuration: MediaConfigurationType, url: urls.URL): string[] {
197      const redirectURLParams = configuration.redirectUrlWhitelistedQueryParams;
198      return redirectURLParams.filter((param) => serverData.isDefinedNonNull(url.query?.[param]));
199  }
200  
201  export type MediaAPIFetchRequest = {
202      url: string;
203      excludeIdentifierHeadersForAccount?: boolean;
204      alwaysIncludeAuthKitHeaders?: boolean;
205      alwaysIncludeMMeClientInfoAndDeviceHeaders?: boolean;
206      method?: Opt<HTTPMethod>;
207      cache?: Opt<HTTPCachePolicy>;
208      signingStyle?: Opt<HTTPSigningStyle>;
209      headers?: Opt<HTTPHeaders>;
210      timeout?: Opt<number>;
211      body?: Opt<string>;
212  };
213  
214  /**
215   * Given a built URL, token, and options, calls into native networking APIs to fetch content.
216   *
217   * @param {string} url URL to fetch data from.
218   * @param {string} token MAPI token key.
219   * @param {FetchOptions} options Fetch options for MAPI requests.
220   * @param {boolean} isRetry flag indicating whether this is a fetch retry following a 401 request, and media token was refreshed.
221   * @returns {Promise<Type>} Promise resolving to some type for given MAPI request.
222   */
223  async function fetchWithToken<Type>(
224      configuration: MediaConfigurationType,
225      mediaToken: MediaTokenService,
226      url: string,
227      token: string,
228      options: FetchOptions = {},
229      isRetry = false,
230      fetchTimingMetricsBuilder: Opt<FetchTimingMetricsBuilder>,
231  ): Promise<Type & ParsedNetworkResponse> {
232      // Removes all affiliate/redirect params for caching (https://connectme.apple.com/docs/DOC-577671)
233      const filteredURL = new urls.URL(url);
234      const redirectParameters = redirectParametersInUrl(configuration, filteredURL);
235      for (const param of redirectParameters) {
236          filteredURL.removeParam(param);
237      }
238      const filteredUrlString = filteredURL.toString();
239  
240      let headers = options.headers;
241      if (headers == null) {
242          headers = {};
243      }
244      headers["Authorization"] = "Bearer " + token;
245  
246      const fetchRequest: MediaAPIFetchRequest = {
247          url: filteredUrlString,
248          excludeIdentifierHeadersForAccount: options.excludeIdentifierHeadersForAccount ?? false,
249          alwaysIncludeAuthKitHeaders: options.alwaysIncludeAuthKitHeaders ?? true,
250          alwaysIncludeMMeClientInfoAndDeviceHeaders: options.alwaysIncludeMMeClientInfoAndDeviceHeaders ?? true,
251          headers: headers,
252          method: options.method,
253          body: options.requestBodyString,
254          timeout: options.timeout,
255      };
256  
257      const response = await configuration.network.fetch(fetchRequest);
258  
259      try {
260          if (response.status === 401 || response.status === 403) {
261              if (isRetry) {
262                  throw Error("We refreshed the token but we still get 401 from the API");
263              }
264              mediaToken.resetToken();
265              return await mediaToken.refreshToken().then(async (newToken) => {
266                  // Explicitly re-fetch with the original request so logging and metrics are correct
267                  return await fetchWithToken<Type>(
268                      configuration,
269                      mediaToken,
270                      url,
271                      newToken,
272                      options,
273                      true,
274                      fetchTimingMetricsBuilder,
275                  );
276              });
277          } else if (response.status === 404) {
278              // item is not available in this storefront or perhaps not at all
279              throw noContentError();
280          } else if (!response.ok) {
281              const correlationKey = response.headers["x-apple-jingle-correlation-key"] ?? "N/A";
282              const error = new NetworkError(
283                  `Bad Status code ${response.status} (correlationKey: ${correlationKey}) for ${filteredUrlString}, original ${url}`,
284              );
285              error.statusCode = response.status;
286              throw error;
287          }
288  
289          const parser = (resp: FetchResponse) => {
290              const parseStartTime = Date.now();
291              let result: Type & ParsedNetworkResponse;
292              if (serverData.isNull(resp.body) || resp.body === "") {
293                  if (resp.status === 204) {
294                      // 204 indicates a success, but the response will typically be empty
295                      // Create a fake result so that we don't throw an error when JSON parsing
296                      const emptyData: ParsedNetworkResponse = {};
297                      result = emptyData as Type & ParsedNetworkResponse;
298                  } else {
299                      throw noContentError();
300                  }
301              } else {
302                  result = JSON.parse(resp.body) as Type & ParsedNetworkResponse;
303              }
304              const parseEndTime = Date.now();
305  
306              result[ResponseMetadata.pageInformation] = serverData.asJSONData(
307                  getPageInformationFromResponse(configuration, resp),
308              );
309              if (resp.metrics.length > 0) {
310                  const metrics: FetchTimingMetrics = {
311                      ...resp.metrics[0],
312                      parseStartTime: parseStartTime,
313                      parseEndTime: parseEndTime,
314                  };
315                  result[ResponseMetadata.timingValues] = metrics;
316              } else {
317                  const fallbackMetrics: FetchTimingMetrics = {
318                      pageURL: resp.url,
319                      parseStartTime,
320                      parseEndTime,
321                  };
322                  result[ResponseMetadata.timingValues] = fallbackMetrics;
323              }
324              result[ResponseMetadata.contentMaxAge] = getContentTimeToLiveFromResponse(resp);
325  
326              // If we have an empty data object, throw a 204 (No Content).
327              if (
328                  Array.isArray(result.data) &&
329                  serverData.isArrayDefinedNonNullAndEmpty(result.data) &&
330                  !serverData.asBooleanOrFalse(options.allowEmptyDataResponse)
331              ) {
332                  throw noContentError();
333              }
334  
335              result[ResponseMetadata.requestedUrl] = url;
336              return result;
337          };
338          if (isSome(fetchTimingMetricsBuilder)) {
339              return fetchTimingMetricsBuilder.measureParsing(response, parser);
340          } else {
341              return parser(response);
342          }
343      } catch (e) {
344          if (e instanceof NetworkError) {
345              throw e;
346          }
347          throw new Error(`Error Fetching - filtered: ${filteredUrlString}, original: ${url}, ${e.name}, ${e.message}`);
348      }
349  }
350  
351  export class NetworkError extends Error {
352      statusCode?: number;
353  }
354  
355  function noContentError(): NetworkError {
356      const error = new NetworkError(`No content`);
357      error.statusCode = 204;
358      return error;
359  }
360  
361  const serverInstanceHeader = "x-apple-application-instance";
362  
363  const environmentDataCenterHeader = "x-apple-application-site";
364  
365  function getPageInformationFromResponse(
366      configuration: MediaConfigurationType,
367      response: FetchResponse,
368  ): MetricsFields | null {
369      const storeFrontHeader: string = configuration.storefrontIdentifier;
370  
371      let storeFront: Opt<string> = null;
372      if (serverData.isDefinedNonNullNonEmpty(storeFrontHeader)) {
373          const storeFrontHeaderComponents: string[] = storeFrontHeader.split("-");
374          if (serverData.isDefinedNonNullNonEmpty(storeFrontHeaderComponents)) {
375              storeFront = storeFrontHeaderComponents[0];
376          }
377      }
378  
379      return {
380          serverInstance: response.headers[serverInstanceHeader],
381          storeFrontHeader: storeFrontHeader,
382          language: configuration.bagLanguage,
383          storeFront: storeFront,
384          environmentDataCenter: response.headers[environmentDataCenterHeader],
385      };
386  }
387  
388  function getContentTimeToLiveFromResponse(response: FetchResponse): Opt<number> {
389      const cacheControlHeaderKey = Object.keys(response.headers).find((key) => key.toLowerCase() === "cache-control");
390      if (serverData.isNull(cacheControlHeaderKey) || cacheControlHeaderKey === "") {
391          return null;
392      }
393  
394      const headerValue = response.headers[cacheControlHeaderKey];
395      if (serverData.isNullOrEmpty(headerValue)) {
396          return null;
397      }
398      const matches = headerValue.match(/max-age=(\d+)/);
399      if (serverData.isNull(matches) || matches.length < 2) {
400          return null;
401      }
402      return serverData.asNumber(matches[1]);
403  }