1  (local fennel (require :fennel)) 
  2  (fn _G.pp [x] (print (fennel.view x)))
  3  
  4  (local block-grammar 
  5    [{:name "Section"
  6      :sp-chars ["-----"]
  7      :regex "%-%-%-%-%-"}
  8     {:name "Image"
  9      :sp-chars ["!["]
 10      :regex "!%[.-%]%(.-%)\n"}
 11     {:name "Tag"
 12      :sp-chars [":"]
 13      :end-also true
 14      :regex ":.-:\n"}
 15     {:name "Quote"
 16      :any-space true
 17      :sp-chars [">"]
 18      :continue-until-no-match true
 19      :regex "%s*>.-\n"}
 20     {:name "Metadata"
 21      :sp-chars ["%"]
 22      :regex "%%.-%s.-\n"}
 23     {:name "Code"
 24      :sp-chars ["{{{"]
 25      :regex "{{{.-\n"
 26      :capture-mode true
 27      :mode-regex "}}}"}
 28     {:name "Header"
 29      :sp-chars ["="]
 30      :end-also true
 31      :regex "=+[^=]-=+\n"} 
 32     {:name "List"
 33      :sp-chars ["#" "-"]
 34      :any-space true
 35      :continue-until-no-match true
 36      :regex "^[%s%-%#].-\n"}
 37     {:name "Footnote"
 38      :sp-chars ["::\n"]
 39      :regex "%s*::%s-\n"
 40      :capture-mode true
 41      :mode-regex "%s*::%s-\n"}
 42     {:name "Paragraph"
 43      :regex ".-\n"}])
 44  
 45  (fn block-lexer [contents grammar ?debug]
 46    (var d (or ?debug false))
 47    (var current 1)
 48    (var blocks [])
 49    (var current-block-type nil)
 50    (var current-block "")
 51    (var capture-mode false)
 52    (while (< current (- (length contents) 1))
 53      (var matched false)
 54      (let [start (string.find contents "\n" current)
 55            current-str (string.sub contents current start)]
 56        (each [_ g (ipairs grammar) &until matched]
 57          (if d
 58            (do
 59              (print (.. "Current: " current))
 60              (print (.. "Start: " start))
 61              (print (.. "Current-str: " current-str))
 62              (print (.. "checking regex: " (if (not capture-mode) 
 63                                              (. g :regex)
 64                                              (tostring (. g :mode-regex)))))))
 65          (let [m (if (not capture-mode)
 66                    (string.match current-str (. g :regex)) ; We're not in previous capture mode
 67                    (if (= (. g :name) current-block-type)  ; And the regex we're checking is the one we want
 68                      (string.match current-str (. g :mode-regex)) ; Use the mode specific regex
 69                      nil))] ; Or don't check
 70            (if d
 71              (print (.. "M: " (tostring m))))
 72            (if (not (= nil m)) ; This is a match or move on
 73              (do
 74                (set matched true)
 75                (if 
 76                  capture-mode ; We matched and it's capture-mode, so we end capture mode
 77                    (do
 78                      (set current-block (.. current-block current-str "\n"))
 79                      (table.insert blocks {:type current-block-type 
 80                                            :block current-block})
 81                      (set current-block "")
 82                      (if d
 83                        (do
 84                          (print "Capture-mode completed")
 85                          (print (.. "Inserted into table: " current-block))
 86                          (print (.. "New current-block: " current-block))))
 87                      (set capture-mode nil)
 88                      (set current-block-type nil))
 89                  (or ; We append to the current block
 90                      (= nil current-block-type) ; If it's the first block
 91                      (and (= (. g :name) current-block-type) ; Or it matches the previous block,
 92                           (not (= (. g :name) "Paragraph")))) ; and it's not a paragraph
 93                  (do ; Continueing adding things
 94                    (set current-block-type (. g :name))
 95                    (set capture-mode (. g :capture-mode))
 96                    (set current-block (.. current-block current-str "\n"))
 97                    (if d
 98                      (print (.. "Set current-block: " current-block))))
 99                  (do ; Else we start a new block, and insert our accumulation
100                    (table.insert blocks {:type current-block-type 
101                                          :block current-block})
102                    (set current-block current-str)
103                    (set capture-mode (. g :capture-mode))
104                    (if d
105                      (do
106                        (print (.. "Inserted into table: " current-block))
107                        (print (.. "New current-block: " current-block))
108                        (print (.. "New Capture-mode: " (tostring (. g :capture-mode))))))
109                    (set current-block-type (. g :name))))
110                (set current (+ 1 start)))
111              (if (and ; We didn't match, BUT
112                    capture-mode ; Or we're in capture mode 
113                    (= (. g :name) current-block-type)) ; And are testing the right regex (should be implicitly true if we reached here)
114                (do 
115                  (set current-block (.. current-block current-str "\n"))
116                  (set matched true) ; We don't need to check the others
117                  (set current (+ 1 start))
118                  (if d
119                    (do
120                      (print (.. "Didnt match capture-mode regex, appending anyway"))
121                      (print (.. "New current-block: " current-block)))))))))))
122    (if current-block-type
123     (do 
124      (if d 
125        (print "End of file reached, unloading current"))
126      (table.insert blocks {:type current-block-type 
127                            :block current-block})))
128    blocks) 
129  
130    ; Each \n point
131    ; Read contents from current to next \n
132    ; If it matches the regex
133    ; Either it's the same pattern as the last one (or the last one was nil), so we continue adding to the block-accumulator
134    ; Or it's a different pattern, which means the current block is complete add it to the blocks list, and start a new accumulator filled with our current-str
135    ; If not, continue with block descent
136  
137  ; This string has weird behaviour in nvim conjure repl 
138  ; (string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n")
139  
140  ; (= [{:type "Paragraph"
141  ;      :block "This is a paragraph."
142  ;     {:type "List"
143  ;      :block "- This\n- List\n- Should\n- Be One"} 
144  
145  {:grammar block-grammar
146   :lexer block-lexer}