ParserAbstract.php 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501
  1. <?php
  2. namespace PhpParser;
  3. /*
  4. * This parser is based on a skeleton written by Moriyoshi Koizumi, which in
  5. * turn is based on work by Masato Bito.
  6. */
  7. use PhpParser\Node\Name;
  8. abstract class ParserAbstract implements Parser
  9. {
  10. const SYMBOL_NONE = -1;
  11. /*
  12. * The following members will be filled with generated parsing data:
  13. */
  14. /** @var int Size of $tokenToSymbol map */
  15. protected $tokenToSymbolMapSize;
  16. /** @var int Size of $action table */
  17. protected $actionTableSize;
  18. /** @var int Size of $goto table */
  19. protected $gotoTableSize;
  20. /** @var int Symbol number signifying an invalid token */
  21. protected $invalidSymbol;
  22. /** @var int Symbol number of error recovery token */
  23. protected $errorSymbol;
  24. /** @var int Action number signifying default action */
  25. protected $defaultAction;
  26. /** @var int Rule number signifying that an unexpected token was encountered */
  27. protected $unexpectedTokenRule;
  28. protected $YY2TBLSTATE;
  29. protected $YYNLSTATES;
  30. /** @var array Map of lexer tokens to internal symbols */
  31. protected $tokenToSymbol;
  32. /** @var array Map of symbols to their names */
  33. protected $symbolToName;
  34. /** @var array Names of the production rules (only necessary for debugging) */
  35. protected $productions;
  36. /** @var array Map of states to a displacement into the $action table. The corresponding action for this
  37. * state/symbol pair is $action[$actionBase[$state] + $symbol]. If $actionBase[$state] is 0, the
  38. action is defaulted, i.e. $actionDefault[$state] should be used instead. */
  39. protected $actionBase;
  40. /** @var array Table of actions. Indexed according to $actionBase comment. */
  41. protected $action;
  42. /** @var array Table indexed analogously to $action. If $actionCheck[$actionBase[$state] + $symbol] != $symbol
  43. * then the action is defaulted, i.e. $actionDefault[$state] should be used instead. */
  44. protected $actionCheck;
  45. /** @var array Map of states to their default action */
  46. protected $actionDefault;
  47. /** @var array Map of non-terminals to a displacement into the $goto table. The corresponding goto state for this
  48. * non-terminal/state pair is $goto[$gotoBase[$nonTerminal] + $state] (unless defaulted) */
  49. protected $gotoBase;
  50. /** @var array Table of states to goto after reduction. Indexed according to $gotoBase comment. */
  51. protected $goto;
  52. /** @var array Table indexed analogously to $goto. If $gotoCheck[$gotoBase[$nonTerminal] + $state] != $nonTerminal
  53. * then the goto state is defaulted, i.e. $gotoDefault[$nonTerminal] should be used. */
  54. protected $gotoCheck;
  55. /** @var array Map of non-terminals to the default state to goto after their reduction */
  56. protected $gotoDefault;
  57. /** @var array Map of rules to the non-terminal on their left-hand side, i.e. the non-terminal to use for
  58. * determining the state to goto after reduction. */
  59. protected $ruleToNonTerminal;
  60. /** @var array Map of rules to the length of their right-hand side, which is the number of elements that have to
  61. * be popped from the stack(s) on reduction. */
  62. protected $ruleToLength;
  63. /*
  64. * The following members are part of the parser state:
  65. */
  66. /** @var Lexer Lexer that is used when parsing */
  67. protected $lexer;
  68. /** @var mixed Temporary value containing the result of last semantic action (reduction) */
  69. protected $semValue;
  70. /** @var int Position in stacks (state stack, semantic value stack, attribute stack) */
  71. protected $stackPos;
  72. /** @var array Semantic value stack (contains values of tokens and semantic action results) */
  73. protected $semStack;
  74. /** @var array[] Start attribute stack */
  75. protected $startAttributeStack;
  76. /** @var array End attributes of last *shifted* token */
  77. protected $endAttributes;
  78. /** @var array Start attributes of last *read* token */
  79. protected $lookaheadStartAttributes;
  80. /** @var bool Whether to throw on first error */
  81. protected $throwOnError;
  82. /** @var Error[] Errors collected during last parse */
  83. protected $errors;
  84. /**
  85. * Creates a parser instance.
  86. *
  87. * @param Lexer $lexer A lexer
  88. * @param array $options Options array. The boolean 'throwOnError' option determines whether an exception should be
  89. * thrown on first error, or if the parser should try to continue parsing the remaining code
  90. * and build a partial AST.
  91. */
  92. public function __construct(Lexer $lexer, array $options = array()) {
  93. $this->lexer = $lexer;
  94. $this->errors = array();
  95. $this->throwOnError = isset($options['throwOnError']) ? $options['throwOnError'] : true;
  96. }
  97. /**
  98. * Get array of errors that occurred during the last parse.
  99. *
  100. * This method may only return multiple errors if the 'throwOnError' option is disabled.
  101. *
  102. * @return Error[]
  103. */
  104. public function getErrors() {
  105. return $this->errors;
  106. }
  107. /**
  108. * Parses PHP code into a node tree.
  109. *
  110. * @param string $code The source code to parse
  111. *
  112. * @return Node[]|null Array of statements (or null if the 'throwOnError' option is disabled and the parser was
  113. * unable to recover from an error).
  114. */
  115. public function parse($code) {
  116. $this->lexer->startLexing($code);
  117. $this->errors = array();
  118. // We start off with no lookahead-token
  119. $symbol = self::SYMBOL_NONE;
  120. // The attributes for a node are taken from the first and last token of the node.
  121. // From the first token only the startAttributes are taken and from the last only
  122. // the endAttributes. Both are merged using the array union operator (+).
  123. $startAttributes = '*POISON';
  124. $endAttributes = '*POISON';
  125. $this->endAttributes = $endAttributes;
  126. // In order to figure out the attributes for the starting token, we have to keep
  127. // them in a stack
  128. $this->startAttributeStack = array();
  129. // Start off in the initial state and keep a stack of previous states
  130. $state = 0;
  131. $stateStack = array($state);
  132. // Semantic value stack (contains values of tokens and semantic action results)
  133. $this->semStack = array();
  134. // Current position in the stack(s)
  135. $this->stackPos = 0;
  136. $errorState = 0;
  137. for (;;) {
  138. //$this->traceNewState($state, $symbol);
  139. if ($this->actionBase[$state] == 0) {
  140. $rule = $this->actionDefault[$state];
  141. } else {
  142. if ($symbol === self::SYMBOL_NONE) {
  143. // Fetch the next token id from the lexer and fetch additional info by-ref.
  144. // The end attributes are fetched into a temporary variable and only set once the token is really
  145. // shifted (not during read). Otherwise you would sometimes get off-by-one errors, when a rule is
  146. // reduced after a token was read but not yet shifted.
  147. $tokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $endAttributes);
  148. // map the lexer token id to the internally used symbols
  149. $symbol = $tokenId >= 0 && $tokenId < $this->tokenToSymbolMapSize
  150. ? $this->tokenToSymbol[$tokenId]
  151. : $this->invalidSymbol;
  152. if ($symbol === $this->invalidSymbol) {
  153. throw new \RangeException(sprintf(
  154. 'The lexer returned an invalid token (id=%d, value=%s)',
  155. $tokenId, $tokenValue
  156. ));
  157. }
  158. // This is necessary to assign some meaningful attributes to /* empty */ productions. They'll get
  159. // the attributes of the next token, even though they don't contain it themselves.
  160. $this->startAttributeStack[$this->stackPos+1] = $startAttributes;
  161. $this->lookaheadStartAttributes = $startAttributes;
  162. //$this->traceRead($symbol);
  163. }
  164. $idx = $this->actionBase[$state] + $symbol;
  165. if ((($idx >= 0 && $idx < $this->actionTableSize && $this->actionCheck[$idx] == $symbol)
  166. || ($state < $this->YY2TBLSTATE
  167. && ($idx = $this->actionBase[$state + $this->YYNLSTATES] + $symbol) >= 0
  168. && $idx < $this->actionTableSize && $this->actionCheck[$idx] == $symbol))
  169. && ($action = $this->action[$idx]) != $this->defaultAction) {
  170. /*
  171. * >= YYNLSTATES: shift and reduce
  172. * > 0: shift
  173. * = 0: accept
  174. * < 0: reduce
  175. * = -YYUNEXPECTED: error
  176. */
  177. if ($action > 0) {
  178. /* shift */
  179. //$this->traceShift($symbol);
  180. ++$this->stackPos;
  181. $stateStack[$this->stackPos] = $state = $action;
  182. $this->semStack[$this->stackPos] = $tokenValue;
  183. $this->startAttributeStack[$this->stackPos] = $startAttributes;
  184. $this->endAttributes = $endAttributes;
  185. $symbol = self::SYMBOL_NONE;
  186. if ($errorState) {
  187. --$errorState;
  188. }
  189. if ($action < $this->YYNLSTATES) {
  190. continue;
  191. }
  192. /* $yyn >= YYNLSTATES means shift-and-reduce */
  193. $rule = $action - $this->YYNLSTATES;
  194. } else {
  195. $rule = -$action;
  196. }
  197. } else {
  198. $rule = $this->actionDefault[$state];
  199. }
  200. }
  201. for (;;) {
  202. if ($rule === 0) {
  203. /* accept */
  204. //$this->traceAccept();
  205. return $this->semValue;
  206. } elseif ($rule !== $this->unexpectedTokenRule) {
  207. /* reduce */
  208. //$this->traceReduce($rule);
  209. try {
  210. $this->{'reduceRule' . $rule}();
  211. } catch (Error $e) {
  212. if (-1 === $e->getStartLine() && isset($startAttributes['startLine'])) {
  213. $e->setStartLine($startAttributes['startLine']);
  214. }
  215. $this->errors[] = $e;
  216. if ($this->throwOnError) {
  217. throw $e;
  218. } else {
  219. // Currently can't recover from "special" errors
  220. return null;
  221. }
  222. }
  223. /* Goto - shift nonterminal */
  224. $this->stackPos -= $this->ruleToLength[$rule];
  225. $nonTerminal = $this->ruleToNonTerminal[$rule];
  226. $idx = $this->gotoBase[$nonTerminal] + $stateStack[$this->stackPos];
  227. if ($idx >= 0 && $idx < $this->gotoTableSize && $this->gotoCheck[$idx] == $nonTerminal) {
  228. $state = $this->goto[$idx];
  229. } else {
  230. $state = $this->gotoDefault[$nonTerminal];
  231. }
  232. ++$this->stackPos;
  233. $stateStack[$this->stackPos] = $state;
  234. $this->semStack[$this->stackPos] = $this->semValue;
  235. } else {
  236. /* error */
  237. switch ($errorState) {
  238. case 0:
  239. $msg = $this->getErrorMessage($symbol, $state);
  240. $error = new Error($msg, $startAttributes + $endAttributes);
  241. $this->errors[] = $error;
  242. if ($this->throwOnError) {
  243. throw $error;
  244. }
  245. // Break missing intentionally
  246. case 1:
  247. case 2:
  248. $errorState = 3;
  249. // Pop until error-expecting state uncovered
  250. while (!(
  251. (($idx = $this->actionBase[$state] + $this->errorSymbol) >= 0
  252. && $idx < $this->actionTableSize && $this->actionCheck[$idx] == $this->errorSymbol)
  253. || ($state < $this->YY2TBLSTATE
  254. && ($idx = $this->actionBase[$state + $this->YYNLSTATES] + $this->errorSymbol) >= 0
  255. && $idx < $this->actionTableSize && $this->actionCheck[$idx] == $this->errorSymbol)
  256. ) || ($action = $this->action[$idx]) == $this->defaultAction) { // Not totally sure about this
  257. if ($this->stackPos <= 0) {
  258. // Could not recover from error
  259. return null;
  260. }
  261. $state = $stateStack[--$this->stackPos];
  262. //$this->tracePop($state);
  263. }
  264. //$this->traceShift($this->errorSymbol);
  265. $stateStack[++$this->stackPos] = $state = $action;
  266. break;
  267. case 3:
  268. if ($symbol === 0) {
  269. // Reached EOF without recovering from error
  270. return null;
  271. }
  272. //$this->traceDiscard($symbol);
  273. $symbol = self::SYMBOL_NONE;
  274. break 2;
  275. }
  276. }
  277. if ($state < $this->YYNLSTATES) {
  278. break;
  279. }
  280. /* >= YYNLSTATES means shift-and-reduce */
  281. $rule = $state - $this->YYNLSTATES;
  282. }
  283. }
  284. throw new \RuntimeException('Reached end of parser loop');
  285. }
  286. protected function getErrorMessage($symbol, $state) {
  287. $expectedString = '';
  288. if ($expected = $this->getExpectedTokens($state)) {
  289. $expectedString = ', expecting ' . implode(' or ', $expected);
  290. }
  291. return 'Syntax error, unexpected ' . $this->symbolToName[$symbol] . $expectedString;
  292. }
  293. protected function getExpectedTokens($state) {
  294. $expected = array();
  295. $base = $this->actionBase[$state];
  296. foreach ($this->symbolToName as $symbol => $name) {
  297. $idx = $base + $symbol;
  298. if ($idx >= 0 && $idx < $this->actionTableSize && $this->actionCheck[$idx] === $symbol
  299. || $state < $this->YY2TBLSTATE
  300. && ($idx = $this->actionBase[$state + $this->YYNLSTATES] + $symbol) >= 0
  301. && $idx < $this->actionTableSize && $this->actionCheck[$idx] === $symbol
  302. ) {
  303. if ($this->action[$idx] != $this->unexpectedTokenRule
  304. && $this->action[$idx] != $this->defaultAction
  305. ) {
  306. if (count($expected) == 4) {
  307. /* Too many expected tokens */
  308. return array();
  309. }
  310. $expected[] = $name;
  311. }
  312. }
  313. }
  314. return $expected;
  315. }
  316. /*
  317. * Tracing functions used for debugging the parser.
  318. */
  319. /*
  320. protected function traceNewState($state, $symbol) {
  321. echo '% State ' . $state
  322. . ', Lookahead ' . ($symbol == self::SYMBOL_NONE ? '--none--' : $this->symbolToName[$symbol]) . "\n";
  323. }
  324. protected function traceRead($symbol) {
  325. echo '% Reading ' . $this->symbolToName[$symbol] . "\n";
  326. }
  327. protected function traceShift($symbol) {
  328. echo '% Shift ' . $this->symbolToName[$symbol] . "\n";
  329. }
  330. protected function traceAccept() {
  331. echo "% Accepted.\n";
  332. }
  333. protected function traceReduce($n) {
  334. echo '% Reduce by (' . $n . ') ' . $this->productions[$n] . "\n";
  335. }
  336. protected function tracePop($state) {
  337. echo '% Recovering, uncovered state ' . $state . "\n";
  338. }
  339. protected function traceDiscard($symbol) {
  340. echo '% Discard ' . $this->symbolToName[$symbol] . "\n";
  341. }
  342. */
  343. /*
  344. * Helper functions invoked by semantic actions
  345. */
  346. /**
  347. * Moves statements of semicolon-style namespaces into $ns->stmts and checks various error conditions.
  348. *
  349. * @param Node[] $stmts
  350. * @return Node[]
  351. */
  352. protected function handleNamespaces(array $stmts) {
  353. $style = $this->getNamespacingStyle($stmts);
  354. if (null === $style) {
  355. // not namespaced, nothing to do
  356. return $stmts;
  357. } elseif ('brace' === $style) {
  358. // For braced namespaces we only have to check that there are no invalid statements between the namespaces
  359. $afterFirstNamespace = false;
  360. foreach ($stmts as $stmt) {
  361. if ($stmt instanceof Node\Stmt\Namespace_) {
  362. $afterFirstNamespace = true;
  363. } elseif (!$stmt instanceof Node\Stmt\HaltCompiler && $afterFirstNamespace) {
  364. throw new Error('No code may exist outside of namespace {}', $stmt->getLine());
  365. }
  366. }
  367. return $stmts;
  368. } else {
  369. // For semicolon namespaces we have to move the statements after a namespace declaration into ->stmts
  370. $resultStmts = array();
  371. $targetStmts =& $resultStmts;
  372. foreach ($stmts as $stmt) {
  373. if ($stmt instanceof Node\Stmt\Namespace_) {
  374. $stmt->stmts = array();
  375. $targetStmts =& $stmt->stmts;
  376. $resultStmts[] = $stmt;
  377. } elseif ($stmt instanceof Node\Stmt\HaltCompiler) {
  378. // __halt_compiler() is not moved into the namespace
  379. $resultStmts[] = $stmt;
  380. } else {
  381. $targetStmts[] = $stmt;
  382. }
  383. }
  384. return $resultStmts;
  385. }
  386. }
  387. private function getNamespacingStyle(array $stmts) {
  388. $style = null;
  389. $hasNotAllowedStmts = false;
  390. foreach ($stmts as $i => $stmt) {
  391. if ($stmt instanceof Node\Stmt\Namespace_) {
  392. $currentStyle = null === $stmt->stmts ? 'semicolon' : 'brace';
  393. if (null === $style) {
  394. $style = $currentStyle;
  395. if ($hasNotAllowedStmts) {
  396. throw new Error('Namespace declaration statement has to be the very first statement in the script', $stmt->getLine());
  397. }
  398. } elseif ($style !== $currentStyle) {
  399. throw new Error('Cannot mix bracketed namespace declarations with unbracketed namespace declarations', $stmt->getLine());
  400. }
  401. continue;
  402. }
  403. /* declare(), __halt_compiler() and nops can be used before a namespace declaration */
  404. if ($stmt instanceof Node\Stmt\Declare_
  405. || $stmt instanceof Node\Stmt\HaltCompiler
  406. || $stmt instanceof Node\Stmt\Nop) {
  407. continue;
  408. }
  409. /* There may be a hashbang line at the very start of the file */
  410. if ($i == 0 && $stmt instanceof Node\Stmt\InlineHTML && preg_match('/\A#!.*\r?\n\z/', $stmt->value)) {
  411. continue;
  412. }
  413. /* Everything else if forbidden before namespace declarations */
  414. $hasNotAllowedStmts = true;
  415. }
  416. return $style;
  417. }
  418. protected function handleScalarTypes(Name $name) {
  419. $scalarTypes = [
  420. 'bool' => true,
  421. 'int' => true,
  422. 'float' => true,
  423. 'string' => true,
  424. ];
  425. if (!$name->isUnqualified()) {
  426. return $name;
  427. }
  428. $lowerName = strtolower($name->toString());
  429. return isset($scalarTypes[$lowerName]) ? $lowerName : $name;
  430. }
  431. }