Fixed python#29132: Updated shlex to work better with punctuation chars in POSIX mode.

Thanks to Evan_ for the report and patch.

Author: vsajip

Lib/shlex.py

@@ -232,11 +232,6 @@ def read_token(self):
                             break   # emit current token
                         else:
                             continue
-                elif self.posix and nextchar in self.quotes:
-                    self.state = nextchar
-                elif self.posix and nextchar in self.escape:
-                    escapedstate = 'a'
-                    self.state = nextchar
                 elif self.state == 'c':
                     if nextchar in self.punctuation_chars:
                         self.token += nextchar
@@ -245,6 +240,11 @@ def read_token(self):
                             self._pushback_chars.append(nextchar)
                         self.state = ' '
                         break
+                elif self.posix and nextchar in self.quotes:
+                    self.state = nextchar
+                elif self.posix and nextchar in self.escape:
+                    escapedstate = 'a'
+                    self.state = nextchar
                 elif (nextchar in self.wordchars or nextchar in self.quotes
                       or self.whitespace_split):
                     self.token += nextchar

Lib/test/test_shlex.py

@@ -273,6 +273,14 @@ def testPunctuationWithWhitespaceSplit(self):
         # white space
         self.assertEqual(list(s), ['a', '&&', 'b', '||', 'c'])
 
+    def testPunctuationWithPosix(self):
+        """Test that punctuation_chars and posix behave correctly together."""
+        # see Issue #29132
+        s = shlex.shlex('f >"abc"', posix=True, punctuation_chars=True)
+        self.assertEqual(list(s), ['f', '>', 'abc'])
+        s = shlex.shlex('f >\\"abc\\"', posix=True, punctuation_chars=True)
+        self.assertEqual(list(s), ['f', '>', '"abc"'])
+
     def testEmptyStringHandling(self):
         """Test that parsing of empty strings is correctly handled."""
         # see Issue #21999