PKKZ`XQ))LICENSEnuW+ACopyright (c) 2006-2018 Doctrine Project Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PKKZsrc/AbstractLexer.phpnuW+A> */ private array $tokens = []; /** * Current lexer position in input string. */ private int $position = 0; /** * Current peek of current lexer position. */ private int $peek = 0; /** * The next token in the input. * * @var Token|null */ public Token|null $lookahead; /** * The last matched/seen token. * * @var Token|null */ public Token|null $token; /** * Composed regex for input parsing. * * @var non-empty-string|null */ private string|null $regex = null; /** * Sets the input data to be tokenized. * * The Lexer is immediately reset and the new input tokenized. * Any unprocessed tokens from any previous input are lost. * * @param string $input The input to be tokenized. * * @return void */ public function setInput(string $input) { $this->input = $input; $this->tokens = []; $this->reset(); $this->scan($input); } /** * Resets the lexer. * * @return void */ public function reset() { $this->lookahead = null; $this->token = null; $this->peek = 0; $this->position = 0; } /** * Resets the peek pointer to 0. * * @return void */ public function resetPeek() { $this->peek = 0; } /** * Resets the lexer position on the input to the given position. * * @param int $position Position to place the lexical scanner. * * @return void */ public function resetPosition(int $position = 0) { $this->position = $position; } /** * Retrieve the original lexer's input until a given position. * * @return string */ public function getInputUntilPosition(int $position) { return substr($this->input, 0, $position); } /** * Checks whether a given token matches the current lookahead. * * @param T $type * * @return bool * * @psalm-assert-if-true !=null $this->lookahead */ public function isNextToken(int|string|UnitEnum $type) { return $this->lookahead !== null && $this->lookahead->isA($type); } /** * Checks whether any of the given tokens matches the current lookahead. * * @param list $types * * @return bool * * @psalm-assert-if-true !=null $this->lookahead */ public function isNextTokenAny(array $types) { return $this->lookahead !== null && $this->lookahead->isA(...$types); } /** * Moves to the next token in the input string. * * @return bool * * @psalm-assert-if-true !null $this->lookahead */ public function moveNext() { $this->peek = 0; $this->token = $this->lookahead; $this->lookahead = isset($this->tokens[$this->position]) ? $this->tokens[$this->position++] : null; return $this->lookahead !== null; } /** * Tells the lexer to skip input tokens until it sees a token with the given value. * * @param T $type The token type to skip until. * * @return void */ public function skipUntil(int|string|UnitEnum $type) { while ($this->lookahead !== null && ! $this->lookahead->isA($type)) { $this->moveNext(); } } /** * Checks if given value is identical to the given token. * * @return bool */ public function isA(string $value, int|string|UnitEnum $token) { return $this->getType($value) === $token; } /** * Moves the lookahead token forward. * * @return Token|null The next token or NULL if there are no more tokens ahead. */ public function peek() { if (isset($this->tokens[$this->position + $this->peek])) { return $this->tokens[$this->position + $this->peek++]; } return null; } /** * Peeks at the next token, returns it and immediately resets the peek. * * @return Token|null The next token or NULL if there are no more tokens ahead. */ public function glimpse() { $peek = $this->peek(); $this->peek = 0; return $peek; } /** * Scans the input string for tokens. * * @param string $input A query string. * * @return void */ protected function scan(string $input) { if (! isset($this->regex)) { $this->regex = sprintf( '/(%s)|%s/%s', implode(')|(', $this->getCatchablePatterns()), implode('|', $this->getNonCatchablePatterns()), $this->getModifiers(), ); } $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE; $matches = preg_split($this->regex, $input, -1, $flags); if ($matches === false) { // Work around https://bugs.php.net/78122 $matches = [[$input, 0]]; } foreach ($matches as $match) { // Must remain before 'value' assignment since it can change content $firstMatch = $match[0]; $type = $this->getType($firstMatch); $this->tokens[] = new Token( $firstMatch, $type, $match[1], ); } } /** * Gets the literal for a given token. * * @param T $token * * @return int|string */ public function getLiteral(int|string|UnitEnum $token) { if ($token instanceof UnitEnum) { return $token::class . '::' . $token->name; } $className = static::class; $reflClass = new ReflectionClass($className); $constants = $reflClass->getConstants(); foreach ($constants as $name => $value) { if ($value === $token) { return $className . '::' . $name; } } return $token; } /** * Regex modifiers * * @return string */ protected function getModifiers() { return 'iu'; } /** * Lexical catchable patterns. * * @return string[] */ abstract protected function getCatchablePatterns(); /** * Lexical non-catchable patterns. * * @return string[] */ abstract protected function getNonCatchablePatterns(); /** * Retrieve token type. Also processes the token value if necessary. * * @return T|null * * @param-out V $value */ abstract protected function getType(string &$value); } PKKZ( src/Token.phpnuW+Avalue = $value; $this->type = $type; $this->position = $position; } /** @param T ...$types */ public function isA(...$types): bool { return in_array($this->type, $types, true); } } PKKZ6qoo README.mdnuW+A# Doctrine Lexer [![Build Status](https://github.com/doctrine/lexer/workflows/Continuous%20Integration/badge.svg)](https://github.com/doctrine/lexer/actions) Base library for a lexer that can be used in Top-Down, Recursive Descent Parsers. This lexer is used in Doctrine Annotations and in Doctrine ORM (DQL). https://www.doctrine-project.org/projects/lexer.html PKKZR UPGRADE.mdnuW+ANote about upgrading: Doctrine uses static and runtime mechanisms to raise awareness about deprecated code. - Use of `@deprecated` docblock that is detected by IDEs (like PHPStorm) or Static Analysis tools (like Psalm, phpstan) - Use of our low-overhead runtime deprecation API, details: https://github.com/doctrine/deprecations/ # Upgrade to 3.0.0 `Doctrine\Common\Lexer\Token` no longer implements `ArrayAccess`. Parameter type declarations have been added to `Doctrine\Common\Lexer\AbstractLexer` and `Doctrine\Common\Lexer\Token`. You should add both parameter type declarations and return type declarations to your lexers, based on the `@return` phpdoc. # Upgrade to 2.0.0 `AbstractLexer::glimpse()` and `AbstractLexer::peek()` now return instances of `Doctrine\Common\Lexer\Token`, which is an array-like class Using it as an array is deprecated in favor of using properties of that class. Using `count()` on it is deprecated with no replacement. PKKZ < composer.jsonnuW+A{ "name": "doctrine/lexer", "description": "PHP Doctrine Lexer parser library that can be used in Top-Down, Recursive Descent Parsers.", "license": "MIT", "type": "library", "keywords": [ "php", "parser", "lexer", "annotations", "docblock" ], "authors": [ { "name": "Guilherme Blanco", "email": "guilhermeblanco@gmail.com" }, { "name": "Roman Borschel", "email": "roman@code-factory.org" }, { "name": "Johannes Schmitt", "email": "schmittjoh@gmail.com" } ], "homepage": "https://www.doctrine-project.org/projects/lexer.html", "require": { "php": "^8.1" }, "require-dev": { "doctrine/coding-standard": "^12", "phpstan/phpstan": "^1.10", "phpunit/phpunit": "^10.5", "psalm/plugin-phpunit": "^0.18.3", "vimeo/psalm": "^5.21" }, "autoload": { "psr-4": { "Doctrine\\Common\\Lexer\\": "src" } }, "autoload-dev": { "psr-4": { "Doctrine\\Tests\\Common\\Lexer\\": "tests" } }, "config": { "allow-plugins": { "composer/package-versions-deprecated": true, "dealerdirect/phpcodesniffer-composer-installer": true }, "sort-packages": true } } PKKZ`XQ))LICENSEnuW+APKKZ`src/AbstractLexer.phpnuW+APKKZ( !src/Token.phpnuW+APKKZ6qoo %README.mdnuW+APKKZR 'UPGRADE.mdnuW+APKKZ < +composer.jsonnuW+APK_1