Various pattern matching fixes

Found by building the translator and using the parser gem's test suite.

Author: kddnewton

lib/syntax_tree/node.rb

@@ -35,6 +35,21 @@ def to(other)
       )
     end
 
+    def deconstruct
+      [start_line, start_char, start_column, end_line, end_char, end_column]
+    end
+
+    def deconstruct_keys(keys)
+      {
+        start_line: start_line,
+        start_char: start_char,
+        start_column: start_column,
+        end_line: end_line,
+        end_char: end_char,
+        end_column: end_column
+      }
+    end
+
     def self.token(line:, char:, column:, size:)
       new(
         start_line: line,
@@ -4334,16 +4349,20 @@ class Heredoc < Node
     # [String] the ending of the heredoc
     attr_reader :ending
 
+    # [Integer] how far to dedent the heredoc
+    attr_reader :dedent
+
     # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the
     # heredoc string literal
     attr_reader :parts
 
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(beginning:, ending: nil, parts: [], location:, comments: [])
+    def initialize(beginning:, ending: nil, dedent: 0, parts: [], location:, comments: [])
       @beginning = beginning
       @ending = ending
+      @dedent = dedent
       @parts = parts
       @location = location
       @comments = comments
@@ -4538,7 +4557,12 @@ def format(q)
       parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest
 
       contents = -> do
-        q.seplist(parts) { |part| q.format(part, stackable: false) }
+        q.group { q.seplist(parts) { |part| q.format(part, stackable: false) } }
+
+        # If there isn't a constant, and there's a blank keyword_rest, then we
+        # have an plain ** that needs to have a `then` after it in order to
+        # parse correctly on the next parse.
+        q.text(" then") if !constant && keyword_rest && keyword_rest.value.nil?
       end
 
       if constant
@@ -5594,11 +5618,17 @@ class MLHSParen < Node
     # [MLHS | MLHSParen] the contents inside of the parentheses
     attr_reader :contents
 
+    # [boolean] whether or not there is a trailing comma at the end of this
+    # list, which impacts destructuring. It's an attr_accessor so that while
+    # the syntax tree is being built it can be set by its parent node
+    attr_accessor :comma
+
     # [Array[ Comment | EmbDoc ]] the comments attached to this node
     attr_reader :comments
 
-    def initialize(contents:, location:, comments: [])
+    def initialize(contents:, comma: false, location:, comments: [])
       @contents = contents
+      @comma = comma
       @location = location
       @comments = comments
     end
@@ -5622,13 +5652,15 @@ def format(q)
 
       if parent.is_a?(MAssign) || parent.is_a?(MLHSParen)
         q.format(contents)
+        q.text(",") if comma
       else
         q.group(0, "(", ")") do
           q.indent do
             q.breakable("")
             q.format(contents)
           end
 
+          q.text(",") if comma
           q.breakable("")
         end
       end

lib/syntax_tree/parser.rb

@@ -516,12 +516,46 @@ def on_array(contents)
     def on_aryptn(constant, requireds, rest, posts)
       parts = [constant, *requireds, rest, *posts].compact
 
+      # If there aren't any parts (no constant, no positional arguments), then
+      # we're matching an empty array. In this case, we're going to look for the
+      # left and right brackets explicitly. Otherwise, we'll just use the bounds
+      # of the various parts.
+      location =
+        if parts.empty?
+          find_token(LBracket).location.to(find_token(RBracket).location)
+        else
+          parts[0].location.to(parts[-1].location)
+        end
+
+      # If there's the optional then keyword, then we'll delete that and use it
+      # as the end bounds of the location.
+      if token = find_token(Kw, "then", consume: false)
+        tokens.delete(token)
+        location = location.to(token.location)
+      end
+
+      # If there is a plain *, then we're going to fix up the location of it
+      # here because it currently doesn't have anything to use for its precise
+      # location. If we hit a comma, then we've gone too far.
+      if rest.is_a?(VarField) && rest.value.nil?
+        tokens.rindex do |token|
+          case token
+          in Op[value: "*"]
+            rest = VarField.new(value: nil, location: token.location)
+            break
+          in Comma
+            break
+          else
+          end
+        end
+      end
+
       AryPtn.new(
         constant: constant,
         requireds: requireds || [],
         rest: rest,
         posts: posts || [],
-        location: parts[0].location.to(parts[-1].location)
+        location: location
       )
     end
 
@@ -1373,15 +1407,35 @@ def on_float(value)
     #     VarField right
     #   ) -> FndPtn
     def on_fndptn(constant, left, values, right)
-      beginning = constant || find_token(LBracket)
-      ending = find_token(RBracket)
+      # The opening of this find pattern is either going to be a left bracket, a
+      # right left parenthesis, or the left splat. We're going to use this to
+      # determine how to find the closing of the pattern, as well as determining
+      # the location of the node.
+      opening =
+        find_token(LBracket, consume: false) ||
+        find_token(LParen, consume: false) ||
+        left
+
+      # The closing is based on the opening, which is either the matched
+      # punctuation or the right splat.
+      closing =
+        case opening
+        in LBracket
+          tokens.delete(opening)
+          find_token(RBracket)
+        in LParen
+          tokens.delete(opening)
+          find_token(RParen)
+        else
+          right
+        end
 
       FndPtn.new(
         constant: constant,
         left: left,
         values: values,
         right: right,
-        location: beginning.location.to(ending.location)
+        location: (constant || opening).location.to(closing.location)
       )
     end
 
@@ -1468,6 +1522,7 @@ def on_heredoc_dedent(string, width)
       @heredocs[-1] = Heredoc.new(
         beginning: heredoc.beginning,
         ending: heredoc.ending,
+        dedent: width,
         parts: string.parts,
         location: heredoc.location
       )
@@ -1481,6 +1536,7 @@ def on_heredoc_end(value)
       @heredocs[-1] = Heredoc.new(
         beginning: heredoc.beginning,
         ending: value.chomp,
+        dedent: heredoc.dedent,
         parts: heredoc.parts,
         location:
           Location.new(
@@ -1501,12 +1557,23 @@ def on_heredoc_end(value)
     #     (nil | VarField) keyword_rest
     #   ) -> HshPtn
     def on_hshptn(constant, keywords, keyword_rest)
+      # Create an artificial VarField if we find an extra ** on the end
+      if !keyword_rest && (token = find_token(Op, "**", consume: false))
+        tokens.delete(token)
+        keyword_rest = VarField.new(value: nil, location: token.location)
+      end
+
+      # Delete the optional then keyword
+      if token = find_token(Kw, "then", consume: false)
+        tokens.delete(token)
+      end
+
       parts = [constant, *keywords&.flatten(1), keyword_rest].compact
       location =
-        if parts.empty?
-          find_token(LBrace).location.to(find_token(RBrace).location)
-        else
+        if parts.any?
           parts[0].location.to(parts[-1].location)
+        else
+          find_token(LBrace).location.to(find_token(RBrace).location)
         end
 
       HshPtn.new(
@@ -2638,6 +2705,7 @@ def on_string_literal(string)
         Heredoc.new(
           beginning: heredoc.beginning,
           ending: heredoc.ending,
+          dedent: heredoc.dedent,
           parts: string.parts,
           location: heredoc.location
         )
@@ -3190,6 +3258,7 @@ def on_xstring_literal(xstring)
         Heredoc.new(
           beginning: heredoc.beginning,
           ending: heredoc.ending,
+          dedent: heredoc.dedent,
           parts: xstring.parts,
           location: heredoc.location
         )

test/fixtures/fndptn.rb

@@ -1,5 +1,29 @@
 %
 case foo
+in *, bar, * then
+end
+-
+case foo
+in [*, bar, *]
+end
+%
+case foo
+in *, bar, *baz
+end
+-
+case foo
+in [*, bar, *baz]
+end
+%
+case foo
+in *foo, bar, *baz
+end
+-
+case foo
+in [*foo, bar, *baz]
+end
+%
+case foo
 in [*, bar, *]
 end
 %
@@ -38,3 +62,43 @@
 case foo
 in Foo[*foo, bar, *baz]
 end
+%
+case foo
+in Foo(*, bar, *)
+end
+-
+case foo
+in Foo[*, bar, *]
+end
+%
+case foo
+in Foo(*, bar, baz, qux, *)
+end
+-
+case foo
+in Foo[*, bar, baz, qux, *]
+end
+%
+case foo
+in Foo(*foo, bar, *)
+end
+-
+case foo
+in Foo[*foo, bar, *]
+end
+%
+case foo
+in Foo(*, bar, *baz)
+end
+-
+case foo
+in Foo[*, bar, *baz]
+end
+%
+case foo
+in Foo(*foo, bar, *baz)
+end
+-
+case foo
+in Foo[*foo, bar, *baz]
+end

test/fixtures/hshptn.rb

@@ -1,5 +1,9 @@
 %
 case foo
+in ** then
+end
+%
+case foo
 in bar:
 end
 %

test/fixtures/mlhs_paren.rb

@@ -8,3 +8,7 @@
 (foo, bar), baz = baz
 %
 foo, (bar, baz,) = baz
+%
+((foo,)) = bar
+-
+foo, = bar