1  import { Coroutine } from '@bablr/coroutine';
  2  import emptyStack from '@iter-tools/imm-stack';
  3  import {
  4    nodeFlags,
  5    buildReferenceTag,
  6    buildNullTag,
  7    buildOpenNodeTag,
  8    buildLiteralTag,
  9    buildCloseNodeTag,
 10    tokenFlags,
 11  } from './builders.js';
 12  import {
 13    printPrettyCSTML as printPrettyCSTMLFromStream,
 14    printCSTML as printCSTMLFromStream,
 15    printSource as printSourceFromStream,
 16    getStreamIterator,
 17  } from './stream.js';
 18  import {
 19    DoctypeTag,
 20    OpenNodeTag,
 21    CloseNodeTag,
 22    ReferenceTag,
 23    ShiftTag,
 24    GapTag,
 25    NullTag,
 26    ArrayInitializerTag,
 27    LiteralTag,
 28    EmbeddedNode,
 29  } from './symbols.js';
 30  import * as btree from './btree.js';
 31  export * from './builders.js';
 32  export * from './print.js';
 33  import {
 34    add,
 35    get,
 36    TagPath,
 37    Path,
 38    isFragmentNode,
 39    isNullNode,
 40    isGapNode,
 41    getOpenTag,
 42    getCloseTag,
 43  } from './path.js';
 44  
 45  export { add, get, isFragmentNode, isNullNode, isGapNode, getOpenTag, getCloseTag };
 46  
 47  export const buildToken = (language, type, value, attributes = {}) => {
 48    return treeFromStreamSync([
 49      buildOpenNodeTag(tokenFlags, language, type, attributes),
 50      buildLiteralTag(value),
 51      buildCloseNodeTag(),
 52    ]);
 53  };
 54  
 55  const isString = (str) => typeof str === 'string';
 56  
 57  const { isArray } = Array;
 58  const { freeze } = Object;
 59  
 60  export const parseReference = (str) => {
 61    let {
 62      1: name,
 63      2: isArray,
 64      3: index,
 65      4: expressionToken,
 66      5: hasGapToken,
 67    } = /^\s*([.#@]|[a-zA-Z]+)\s*(\[\s*(\d+\s*)?\])?\s*(\+)?(\$)?\s*$/.exec(str);
 68  
 69    let flags = {
 70      expression: !!expressionToken,
 71      hasGap: !!hasGapToken,
 72    };
 73  
 74    index = index ? parseInt(index, 10) : null;
 75    isArray = !!isArray;
 76    name = name || null;
 77  
 78    return buildReferenceTag(name, isArray, flags, index);
 79  };
 80  
 81  export const mergeReferences = (outer, inner) => {
 82    let {
 83      name,
 84      isArray,
 85      index,
 86      flags: { expression, hasGap },
 87    } = outer.value;
 88  
 89    if (
 90      name != null &&
 91      name !== '.' &&
 92      inner.value.name != null &&
 93      inner.value.name !== '.' &&
 94      name !== inner.value.name
 95    )
 96      throw new Error();
 97  
 98    isArray = isArray || inner.value.isArray;
 99    expression = !!(expression || inner.value.flags.expression);
100    hasGap = !!(hasGap || inner.value.flags.hasGap);
101    name = name === '.' ? inner.value.name : name;
102  
103    return buildReferenceTag(name, isArray, { expression, hasGap }, index);
104  };
105  
106  export const isEmptyReference = (ref) => {
107    let { name, isArray, flags } = ref.value;
108    return name === '.' && !isArray && !(flags.expression || flags.hasGap);
109  };
110  
111  function* __treeFromStream(tags, options) {
112    let path = null;
113    let rootPath = null;
114    let held = null;
115    let doctype = null;
116    const co = new Coroutine(getStreamIterator(tags));
117    const expressionsCo = new Coroutine(getStreamIterator(options.expressions || []));
118    let reference = null;
119  
120    for (;;) {
121      co.advance();
122  
123      if (co.current instanceof Promise) {
124        co.current = yield co.current;
125      }
126  
127      if (co.done) break;
128  
129      const tag = co.value;
130  
131      if (tag.type === 'Effect') {
132        continue;
133      }
134  
135      if (tag.type === DoctypeTag) {
136        doctype = tag;
137        continue;
138      }
139  
140      if (held && tag.type !== OpenNodeTag && tag.type !== GapTag) {
141        throw new Error('cannot eat this type of tag while holding');
142      }
143  
144      let suppressTag = false;
145  
146      switch (tag.type) {
147        case LiteralTag:
148        case CloseNodeTag: {
149          break;
150        }
151  
152        case ReferenceTag: {
153          reference = tag;
154          suppressTag = true;
155          break;
156        }
157  
158        case ArrayInitializerTag: {
159          add(path.node, reference, []);
160          suppressTag = true;
161          reference = null;
162          break;
163        }
164  
165        case NullTag:
166        case GapTag: {
167          if (!path) {
168            return buildStubNode(tag);
169          }
170  
171          const isGap = tag.type === GapTag;
172  
173          if (path.parent && reference.type !== ReferenceTag) throw new Error();
174  
175          let node = createNode(tag);
176  
177          if (isGap) {
178            if (held) {
179              node = held;
180              add(path.node, reference, node);
181              suppressTag = true;
182            } else if (!expressionsCo.done) {
183              expressionsCo.advance();
184  
185              let outerReference = reference;
186  
187              if (!expressionsCo.done) {
188                node =
189                  node == null
190                    ? buildStubNode(buildNullTag())
191                    : expressionsCo.value == null
192                    ? buildStubNode(buildNullTag())
193                    : expressionsCo.value;
194                suppressTag = true;
195  
196                if (isFragmentNode(node)) {
197                  const parentNode = path.node;
198  
199                  let reference;
200  
201                  for (const tag of btree.traverse(node.children)) {
202                    switch (tag.type) {
203                      case DoctypeTag: {
204                        break;
205                      }
206                      case OpenNodeTag:
207                      case CloseNodeTag: {
208                        if (!tag.value.type) {
209                          break;
210                        } else {
211                          throw new Error();
212                        }
213                      }
214  
215                      case ReferenceTag:
216                        // combine tags for .
217  
218                        reference = tag;
219                        break;
220  
221                      case ArrayInitializerTag: {
222                        add(parentNode, mergeReferences(outerReference, reference), []);
223                        break;
224                      }
225  
226                      case EmbeddedNode: {
227                        add(parentNode, mergeReferences(outerReference, reference), tag.value);
228                        break;
229                      }
230  
231                      case GapTag: {
232                        const resolvedNode = get(reference, node);
233                        add(parentNode, mergeReferences(outerReference, reference), resolvedNode);
234                        break;
235                      }
236  
237                      case NullTag: {
238                        add(parentNode, mergeReferences(outerReference, reference), null);
239                        break;
240                      }
241  
242                      default:
243                        throw new Error();
244                    }
245                  }
246                } else {
247                  if (path.node.flags.token) {
248                    throw new Error('not implemented');
249                  }
250                  add(path.node, reference, node);
251                }
252              } else {
253                if (!path.node.flags.token) {
254                  add(path.node, reference, node);
255                }
256              }
257            }
258          }
259  
260          reference = null;
261          held = isGap ? null : held;
262  
263          if (!path.node.flags.token) {
264            path = { parent: path, node, depth: (path.depth || -1) + 1, arrays: new Set() };
265          }
266  
267          break;
268        }
269  
270        // case ShiftTag: {
271        //   const { children, properties } = path.node;
272  
273        //   let property = properties[ref.value.name];
274        //   let node;
275  
276        //   if (ref.value.isArray) {
277        //     ({ node } = btree.getAt(-1, property));
278        //     properties[ref.value.name].pop();
279        //   } else {
280        //     ({ node } = property);
281        //     properties[ref.value.name] = null;
282        //   }
283  
284        //   held = node;
285        //   break;
286        // }
287  
288        case OpenNodeTag: {
289          if (path) {
290            const node = createNode(tag);
291  
292            if (path) {
293              add(path.node, reference, node);
294              reference = null;
295            }
296  
297            path = { parent: path, node, depth: (path ? path.depth : -1) + 1, arrays: new Set() };
298          } else {
299            const { language, type, flags, attributes } = tag.value;
300  
301            const attributes_ = doctype?.value.attributes ?? attributes;
302            const language_ = attributes?.['bablrLanguage'] ?? language;
303  
304            const node = {
305              flags,
306              language: language_,
307              type,
308              children: [],
309              properties: {},
310              attributes: attributes_,
311            };
312  
313            path = { parent: null, node, depth: 0, arrays: new Set() };
314  
315            rootPath = path;
316          }
317  
318          break;
319        }
320  
321        default: {
322          throw new Error();
323        }
324      }
325  
326      if (!suppressTag) {
327        path.node.children = btree.push(path.node.children, tag);
328      }
329  
330      switch (tag.type) {
331        case NullTag:
332        case GapTag:
333        case CloseNodeTag: {
334          const completedNode = path.node;
335  
336          if (!(tag.type === GapTag && completedNode.flags.token)) {
337            finalizeNode(completedNode);
338          }
339  
340          if (tag.type === GapTag) {
341            if (path && completedNode.type === null && completedNode.flags.token) {
342              break;
343            }
344          }
345  
346          path = path.parent;
347          break;
348        }
349      }
350    }
351  
352    if (path && path.node.type) {
353      throw new Error('imbalanced tag stack');
354    }
355  
356    return rootPath.node;
357  }
358  
359  export const buildNullNode = () => {
360    return treeFromStreamSync([buildNullTag()]);
361  };
362  
363  export const treeFromStream = (tags, options = {}) => __treeFromStream(tags, options);
364  
365  export const treeFromStreamSync = (tokens, options = {}) => {
366    return evaluateReturnSync(treeFromStream(tokens, options));
367  };
368  
369  export const treeFromStreamAsync = async (tokens, options = {}) => {
370    return evaluateReturnAsync(treeFromStream(tokens, options));
371  };
372  
373  export const evaluateReturnSync = (generator) => {
374    const co = new Coroutine(generator[Symbol.iterator]());
375    while (!co.done) co.advance();
376    return co.value;
377  };
378  
379  export const evaluateReturnAsync = async (generator) => {
380    const co = new Coroutine(getStreamIterator(generator));
381    while (!co.done) {
382      co.advance();
383  
384      if (co.current instanceof Promise) {
385        co.current = await co.current;
386      }
387    }
388    return co.value;
389  };
390  
391  export const streamFromTree = (rootNode, options = {}) => __streamFromTree(rootNode, options);
392  
393  export const isEmpty = (node) => {
394    const { properties } = node;
395  
396    let ref = null;
397  
398    for (const tag of btree.traverse(node.children)) {
399      switch (tag.type) {
400        case ReferenceTag: {
401          const { name } = tag.value;
402  
403          ref = tag;
404  
405          if (properties[name]) {
406            const property = properties[name];
407  
408            if (
409              property != null ||
410              (isArray(property) && property.length) ||
411              !isNullNode(property.node)
412            ) {
413              return false;
414            }
415          }
416          break;
417        }
418  
419        case EmbeddedNode: {
420          if (ref.value.name === '@') {
421            return false;
422          }
423          break;
424        }
425  
426        case LiteralTag:
427        case GapTag:
428          return false;
429      }
430    }
431    return true;
432  };
433  
434  export const buildStubNode = (tag) => {
435    return freeze({
436      flags: nodeFlags,
437      language: null,
438      type: null,
439      children: freeze([tag]),
440      properties: freeze({}),
441      attributes: freeze({}),
442    });
443  };
444  
445  function* __streamFromTree(rootNode, options) {
446    const { unshift = false } = options;
447    if (!rootNode || !btree.getSum(rootNode.children)) return;
448  
449    let tagPath = TagPath.from(Path.from(rootNode), 0);
450  
451    let count = 0;
452  
453    do {
454      if (tagPath.tag.type === OpenNodeTag) count++;
455      if (tagPath.tag.type === CloseNodeTag) count--;
456  
457      yield tagPath.tag;
458    } while ((tagPath = unshift ? tagPath.nextUnshifted : tagPath.next));
459  
460    if (count !== 0) throw new Error();
461  }
462  
463  export const getCooked = (cookable) => {
464    if (!cookable || isGapNode(cookable.type)) {
465      return '';
466    }
467  
468    const children = cookable.children || cookable;
469  
470    let cooked = '';
471  
472    // const openTag = getOpenTag(cookable);
473    // const closeTag = getCloseTag(cookable);
474  
475    let reference = null;
476  
477    for (const tag of btree.traverse(children)) {
478      switch (tag.type) {
479        case ReferenceTag: {
480          const { name } = tag.value;
481  
482          if (!(name === '#' || name === '@')) {
483            throw new Error('cookable nodes must not contain other nodes');
484          }
485  
486          reference = tag;
487          break;
488        }
489  
490        case EmbeddedNode: {
491          const { attributes } = tag.value;
492  
493          if (reference.value.name === '@') {
494            const { cooked: cookedValue } = attributes;
495  
496            if (!isString(cookedValue))
497              throw new Error('cannot cook string: it contains uncooked escapes');
498  
499            cooked += cookedValue;
500          }
501  
502          break;
503        }
504  
505        case LiteralTag: {
506          cooked += tag.value;
507          break;
508        }
509  
510        case OpenNodeTag: {
511          break;
512        }
513  
514        case CloseNodeTag: {
515          break;
516        }
517  
518        default: {
519          throw new Error();
520        }
521      }
522    }
523  
524    return cooked;
525  };
526  
527  export const printCSTML = (rootNode) => {
528    return printCSTMLFromStream(streamFromTree(rootNode));
529  };
530  
531  export const printPrettyCSTML = (rootNode, options = {}) => {
532    return printPrettyCSTMLFromStream(streamFromTree(rootNode), options);
533  };
534  
535  export const printSource = (rootNode) => {
536    return printSourceFromStream(streamFromTree(rootNode, { unshift: true }));
537  };
538  
539  export const sourceTextFor = printSource;
540  
541  export const getRange = (node) => {
542    const { children } = node;
543    return btree.getSum(children) ? [btree.getAt(0, children), btree.getAt(-1, children)] : null;
544  };
545  
546  export const createNode = (openTag) => {
547    if (!openTag || openTag.type === GapTag || openTag.type === NullTag) {
548      return {
549        flags: nodeFlags,
550        language: openTag?.language,
551        type: openTag && ([NullTag, GapTag].includes(openTag.type) ? null : openTag.type),
552        children: [],
553        properties: {},
554        attributes: openTag?.attributes || {},
555      };
556    } else {
557      const { flags, language, type, attributes = {} } = openTag.value || {};
558      return { flags, language, type, children: [], properties: {}, attributes };
559    }
560  };
561  
562  export const finalizeNode = (node) => {
563    for (const property of Object.values(node.properties)) {
564      if (isArray(property)) {
565        btree.freeze(property);
566        for (const childProperty of btree.traverse(property)) {
567          freeze(childProperty);
568          if (childProperty.reference.value.flags.expression) {
569            btree.freeze(childProperty.node);
570          }
571        }
572      } else {
573        freeze(property);
574  
575        if (property.reference.value.flags.expression) {
576          btree.freeze(property.node);
577        }
578      }
579    }
580  
581    freeze(node);
582    btree.freeze(node.children);
583    freeze(node.properties);
584    freeze(node.attributes);
585    return node;
586  };
587  
588  export const notNull = (node) => {
589    return node != null && !isNullNode(node);
590  };
591  
592  export const isNull = (node) => {
593    return node == null || isNullNode(node);
594  };
595  
596  export const branchProperties = (properties) => {
597    const copy = { ...properties };
598  
599    for (const { 0: key, 1: value } of Object.entries(copy)) {
600      if (isArray(value)) {
601        copy[key] = btree.fromValues(value);
602      }
603    }
604  
605    return copy;
606  };
607  
608  export const branchNode = (node) => {
609    const { flags, language, type, children, properties, attributes } = node;
610    return {
611      flags,
612      language,
613      type,
614      // if we always use immutable trees we won't need to do this
615      children: btree.fromValues(btree.traverse(children)),
616      properties: branchProperties(properties),
617      attributes: { ...attributes },
618    };
619  };
620  
621  export const acceptNode = (node, accepted) => {
622    const { children, properties, attributes } = accepted;
623    node.children = children;
624    node.properties = properties;
625    node.attributes = attributes;
626    return node;
627  };
628  
629  export const getRoot = (node) => {
630    return node == null ? node : isFragmentNode(node) ? node.properties['.'].node : node;
631  };
632  
633  export function* traverseProperties(properties) {
634    for (const value of Object.values(properties)) {
635      if (isArray(value)) {
636        for (let item of btree.traverse(value)) {
637          if (isArray(item.node)) {
638            yield btree.getAt(-1, item.node);
639          } else {
640            yield item.node;
641          }
642        }
643      } else {
644        yield value.node;
645      }
646    }
647  }
648  
649  export class Resolver {
650    constructor(
651      states = emptyStack.push({ properties: new Map(), idx: 0 }),
652      reference = null,
653      popped = false,
654      held = null,
655    ) {
656      this.states = states;
657      this.reference = reference;
658      this.popped = popped;
659      this.held = held;
660      this.doctype = null;
661    }
662  
663    get idx() {
664      return this.states.value.idx;
665    }
666  
667    get properties() {
668      return this.states.value.properties;
669    }
670  
671    advance(tag) {
672      const { states } = this;
673  
674      ++states.value.idx;
675  
676      this.popped = false;
677  
678      let hadReference = this.reference;
679  
680      switch (tag.type) {
681        case ReferenceTag: {
682          const { name, isArray } = tag.value;
683          const { properties } = states.value;
684  
685          if (this.reference) throw new Error();
686  
687          this.reference = tag;
688  
689          if (name && name !== '#' && name !== '@') {
690            let state = properties.get(name);
691  
692            if (isArray) {
693              if (state && !state.isArray) throw new Error();
694  
695              const { count = -1 } = state || {};
696  
697              state = { count: count + 1, isArray };
698            } else if (state) {
699              throw new Error(`attempted to consume property {name: ${name}} twice`);
700            } else {
701              state = { count: 1, isArray: false };
702            }
703  
704            properties.set(name, state);
705          }
706  
707          break;
708        }
709  
710        case EmbeddedNode: {
711          if (!this.reference || !['#', '@'].includes(this.reference.value.name)) throw new Error();
712  
713          // this.states = states.push({ properties: new Map(), idx: 0 });
714          break;
715        }
716  
717        case OpenNodeTag: {
718          const { reference } = this;
719          const { flags } = tag.value;
720          const isRootNode = states.size === 1;
721  
722          if (tag.value.type) {
723            this.states = states.push({ properties: new Map(), idx: 0 });
724          }
725  
726          if (!tag.value.type && (!isRootNode || this.reference)) throw new Error();
727  
728          if (
729            tag.type === OpenNodeTag &&
730            ((!reference && !isRootNode) ||
731              (reference &&
732                reference.type !== ReferenceTag &&
733                reference.type !== ShiftTag &&
734                reference.type !== OpenNodeTag))
735          ) {
736            throw new Error('Invalid location for OpenNodeTag');
737          }
738  
739          if (!isRootNode && !reference) {
740            throw new Error();
741          }
742  
743          this.reference = null;
744          break;
745        }
746  
747        case ArrayInitializerTag: {
748          if (!this.reference) throw new Error();
749  
750          const { name } = this.reference.value;
751          const { properties } = states.value;
752          const state = properties.get(name);
753  
754          if (!state || !state.isArray || state.count !== 0) throw new Error();
755  
756          properties.set(name, { count: 0, isArray: true });
757  
758          this.reference = null;
759          break;
760        }
761  
762        case ShiftTag: {
763          this.held = this.states.value;
764          this.states = this.states.push({ properties: new Map(), idx: 0 });
765          this.reference = tag;
766  
767          break;
768        }
769  
770        case NullTag: {
771          if (!this.reference) throw new Error();
772  
773          this.popped = true;
774          this.reference = null;
775          break;
776        }
777  
778        case GapTag: {
779          // if (!this.reference) throw new Error();
780  
781          if (this.held) {
782            // this.states = this.states.push(this.held);
783            this.held = null;
784          }
785  
786          this.popped = true;
787          this.reference = null;
788          break;
789        }
790  
791        case CloseNodeTag: {
792          if (this.reference) throw new Error();
793  
794          this.states = states.pop();
795          this.popped = true;
796          break;
797        }
798  
799        case DoctypeTag:
800          this.doctype = tag;
801          break;
802  
803        case LiteralTag:
804          break;
805  
806        default:
807          throw new Error();
808      }
809  
810      if (hadReference && this.reference) throw new Error();
811  
812      return this;
813    }
814  
815    resolve(reference) {
816      let { name, isArray, flags } = reference.value;
817      const { states } = this;
818      const state = states.value.properties.get(name);
819      let index = null;
820  
821      if (name === '@' || name === '#') return reference;
822  
823      if (isArray && state) {
824        index = state?.count || 0;
825      }
826  
827      return buildReferenceTag(name, isArray, flags, index);
828    }
829  
830    branch() {
831      const { states, reference, popped, held } = this;
832      const { properties, idx } = states.value;
833  
834      return new Resolver(
835        states.replace({ properties: new Map(properties), idx }),
836        reference,
837        popped,
838        held,
839      );
840    }
841  
842    accept(resolver) {
843      this.states = resolver.states;
844      this.reference = resolver.reference;
845      this.popped = resolver.popped;
846      this.held = resolver.held;
847  
848      return this;
849    }
850  }
851  
852  freeze(Resolver.prototype);