1/*
2   +----------------------------------------------------------------------+
3   | Zend Engine                                                          |
4   +----------------------------------------------------------------------+
5   | Copyright (c) Zend Technologies Ltd. (http://www.zend.com)           |
6   +----------------------------------------------------------------------+
7   | This source file is subject to version 2.00 of the Zend license,     |
8   | that is bundled with this package in the file LICENSE, and is        |
9   | available through the world-wide-web at the following url:           |
10   | http://www.zend.com/license/2_00.txt.                                |
11   | If you did not receive a copy of the Zend license and are unable to  |
12   | obtain it through the world-wide-web, please send a note to          |
13   | license@zend.com so we can mail you a copy immediately.              |
14   +----------------------------------------------------------------------+
15   | Authors: Marcus Boerger <helly@php.net>                              |
16   |          Nuno Lopes <nlopess@php.net>                                |
17   |          Scott MacVicar <scottmac@php.net>                           |
18   | Flex version authors:                                                |
19   |          Andi Gutmans <andi@php.net>                                 |
20   |          Zeev Suraski <zeev@php.net>                                 |
21   +----------------------------------------------------------------------+
22*/
23
24#if 0
25# define YYDEBUG(s, c) printf("state: %d char: %c\n", s, c)
26#else
27# define YYDEBUG(s, c)
28#endif
29
30#include "zend_language_scanner_defs.h"
31
32#include <errno.h>
33#include "zend.h"
34#ifdef ZEND_WIN32
35# include <Winuser.h>
36#endif
37#include "zend_alloc.h"
38#include <zend_language_parser.h>
39#include "zend_compile.h"
40#include "zend_language_scanner.h"
41#include "zend_highlight.h"
42#include "zend_constants.h"
43#include "zend_variables.h"
44#include "zend_operators.h"
45#include "zend_API.h"
46#include "zend_strtod.h"
47#include "zend_exceptions.h"
48#include "zend_virtual_cwd.h"
49
50#define YYCTYPE   unsigned char
51#define YYFILL(n) { if ((YYCURSOR + n) >= (YYLIMIT + ZEND_MMAP_AHEAD)) { return 0; } }
52#define YYCURSOR  SCNG(yy_cursor)
53#define YYLIMIT   SCNG(yy_limit)
54#define YYMARKER  SCNG(yy_marker)
55
56#define YYGETCONDITION()  SCNG(yy_state)
57#define YYSETCONDITION(s) SCNG(yy_state) = s
58
59#define STATE(name)  yyc##name
60
61/* emulate flex constructs */
62#define BEGIN(state) YYSETCONDITION(STATE(state))
63#define YYSTATE      YYGETCONDITION()
64#define yytext       ((char*)SCNG(yy_text))
65#define yyleng       SCNG(yy_leng)
66#define yyless(x)    do { YYCURSOR = (unsigned char*)yytext + x; \
67                          yyleng   = (unsigned int)x; } while(0)
68#define yymore()     goto yymore_restart
69
70/* perform sanity check. If this message is triggered you should
71   increase the ZEND_MMAP_AHEAD value in the zend_streams.h file */
72/*!max:re2c */
73#if ZEND_MMAP_AHEAD < YYMAXFILL
74# error ZEND_MMAP_AHEAD should be greater than or equal to YYMAXFILL
75#endif
76
77#include <stdarg.h>
78
79#ifdef HAVE_UNISTD_H
80# include <unistd.h>
81#endif
82
83/* Globals Macros */
84#define SCNG	LANG_SCNG
85#ifdef ZTS
86ZEND_API ts_rsrc_id language_scanner_globals_id;
87ZEND_API size_t language_scanner_globals_offset;
88#else
89ZEND_API zend_php_scanner_globals language_scanner_globals;
90#endif
91
92#define HANDLE_NEWLINES(s, l)													\
93do {																			\
94	char *p = (s), *boundary = p+(l);											\
95																				\
96	while (p<boundary) {														\
97		if (*p == '\n' || (*p == '\r' && (*(p+1) != '\n'))) {					\
98			CG(zend_lineno)++;													\
99		}																		\
100		p++;																	\
101	}																			\
102} while (0)
103
104#define HANDLE_NEWLINE(c) \
105{ \
106	if (c == '\n' || c == '\r') { \
107		CG(zend_lineno)++; \
108	} \
109}
110
111/* To save initial string length after scanning to first variable */
112#define SET_DOUBLE_QUOTES_SCANNED_LENGTH(len) SCNG(scanned_string_len) = (len)
113#define GET_DOUBLE_QUOTES_SCANNED_LENGTH()    SCNG(scanned_string_len)
114
115#define IS_LABEL_START(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z') || (c) == '_' || (c) >= 0x80)
116#define IS_LABEL_SUCCESSOR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z') || ((c) >= '0' && (c) <= '9') || (c) == '_' || (c) >= 0x80)
117
118#define ZEND_IS_OCT(c)  ((c)>='0' && (c)<='7')
119#define ZEND_IS_HEX(c)  (((c)>='0' && (c)<='9') || ((c)>='a' && (c)<='f') || ((c)>='A' && (c)<='F'))
120
121BEGIN_EXTERN_C()
122
123static void strip_underscores(char *str, size_t *len)
124{
125	char *src = str, *dest = str;
126	while (*src != '\0') {
127		if (*src != '_') {
128			*dest = *src;
129			dest++;
130		} else {
131			--(*len);
132		}
133		src++;
134	}
135	*dest = '\0';
136}
137
138static size_t encoding_filter_script_to_internal(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
139{
140	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
141	ZEND_ASSERT(internal_encoding);
142	return zend_multibyte_encoding_converter(to, to_length, from, from_length, internal_encoding, LANG_SCNG(script_encoding));
143}
144
145static size_t encoding_filter_script_to_intermediate(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
146{
147	return zend_multibyte_encoding_converter(to, to_length, from, from_length, zend_multibyte_encoding_utf8, LANG_SCNG(script_encoding));
148}
149
150static size_t encoding_filter_intermediate_to_script(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
151{
152	return zend_multibyte_encoding_converter(to, to_length, from, from_length,
153LANG_SCNG(script_encoding), zend_multibyte_encoding_utf8);
154}
155
156static size_t encoding_filter_intermediate_to_internal(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
157{
158	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
159	ZEND_ASSERT(internal_encoding);
160	return zend_multibyte_encoding_converter(to, to_length, from, from_length,
161internal_encoding, zend_multibyte_encoding_utf8);
162}
163
164
165static void _yy_push_state(int new_state)
166{
167	zend_stack_push(&SCNG(state_stack), (void *) &YYGETCONDITION());
168	YYSETCONDITION(new_state);
169}
170
171#define yy_push_state(state_and_tsrm) _yy_push_state(yyc##state_and_tsrm)
172
173static void yy_pop_state(void)
174{
175	int *stack_state = zend_stack_top(&SCNG(state_stack));
176	YYSETCONDITION(*stack_state);
177	zend_stack_del_top(&SCNG(state_stack));
178}
179
180static void yy_scan_buffer(char *str, unsigned int len)
181{
182	YYCURSOR       = (YYCTYPE*)str;
183	YYLIMIT        = YYCURSOR + len;
184	if (!SCNG(yy_start)) {
185		SCNG(yy_start) = YYCURSOR;
186	}
187}
188
189void startup_scanner(void)
190{
191	CG(parse_error) = 0;
192	CG(doc_comment) = NULL;
193	CG(extra_fn_flags) = 0;
194	zend_stack_init(&SCNG(state_stack), sizeof(int));
195	zend_stack_init(&SCNG(nest_location_stack), sizeof(zend_nest_location));
196	zend_ptr_stack_init(&SCNG(heredoc_label_stack));
197	SCNG(heredoc_scan_ahead) = 0;
198}
199
200static void heredoc_label_dtor(zend_heredoc_label *heredoc_label) {
201    efree(heredoc_label->label);
202}
203
204void shutdown_scanner(void)
205{
206	CG(parse_error) = 0;
207	RESET_DOC_COMMENT();
208	zend_stack_destroy(&SCNG(state_stack));
209	zend_stack_destroy(&SCNG(nest_location_stack));
210	zend_ptr_stack_clean(&SCNG(heredoc_label_stack), (void (*)(void *)) &heredoc_label_dtor, 1);
211	zend_ptr_stack_destroy(&SCNG(heredoc_label_stack));
212	SCNG(heredoc_scan_ahead) = 0;
213	SCNG(on_event) = NULL;
214}
215
216ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state)
217{
218	lex_state->yy_leng   = SCNG(yy_leng);
219	lex_state->yy_start  = SCNG(yy_start);
220	lex_state->yy_text   = SCNG(yy_text);
221	lex_state->yy_cursor = SCNG(yy_cursor);
222	lex_state->yy_marker = SCNG(yy_marker);
223	lex_state->yy_limit  = SCNG(yy_limit);
224
225	lex_state->state_stack = SCNG(state_stack);
226	zend_stack_init(&SCNG(state_stack), sizeof(int));
227
228	lex_state->nest_location_stack = SCNG(nest_location_stack);
229	zend_stack_init(&SCNG(nest_location_stack), sizeof(zend_nest_location));
230
231	lex_state->heredoc_label_stack = SCNG(heredoc_label_stack);
232	zend_ptr_stack_init(&SCNG(heredoc_label_stack));
233
234	lex_state->in = SCNG(yy_in);
235	lex_state->yy_state = YYSTATE;
236	lex_state->filename = CG(compiled_filename);
237	lex_state->lineno = CG(zend_lineno);
238	CG(compiled_filename) = NULL;
239
240	lex_state->script_org = SCNG(script_org);
241	lex_state->script_org_size = SCNG(script_org_size);
242	lex_state->script_filtered = SCNG(script_filtered);
243	lex_state->script_filtered_size = SCNG(script_filtered_size);
244	lex_state->input_filter = SCNG(input_filter);
245	lex_state->output_filter = SCNG(output_filter);
246	lex_state->script_encoding = SCNG(script_encoding);
247
248	lex_state->on_event = SCNG(on_event);
249	lex_state->on_event_context = SCNG(on_event_context);
250
251	lex_state->ast = CG(ast);
252	lex_state->ast_arena = CG(ast_arena);
253}
254
255ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state)
256{
257	SCNG(yy_leng)   = lex_state->yy_leng;
258	SCNG(yy_start)  = lex_state->yy_start;
259	SCNG(yy_text)   = lex_state->yy_text;
260	SCNG(yy_cursor) = lex_state->yy_cursor;
261	SCNG(yy_marker) = lex_state->yy_marker;
262	SCNG(yy_limit)  = lex_state->yy_limit;
263
264	zend_stack_destroy(&SCNG(state_stack));
265	SCNG(state_stack) = lex_state->state_stack;
266
267	zend_stack_destroy(&SCNG(nest_location_stack));
268	SCNG(nest_location_stack) = lex_state->nest_location_stack;
269
270	zend_ptr_stack_clean(&SCNG(heredoc_label_stack), (void (*)(void *)) &heredoc_label_dtor, 1);
271	zend_ptr_stack_destroy(&SCNG(heredoc_label_stack));
272	SCNG(heredoc_label_stack) = lex_state->heredoc_label_stack;
273
274	SCNG(yy_in) = lex_state->in;
275	YYSETCONDITION(lex_state->yy_state);
276	CG(zend_lineno) = lex_state->lineno;
277	zend_restore_compiled_filename(lex_state->filename);
278
279	if (SCNG(script_filtered)) {
280		efree(SCNG(script_filtered));
281		SCNG(script_filtered) = NULL;
282	}
283	SCNG(script_org) = lex_state->script_org;
284	SCNG(script_org_size) = lex_state->script_org_size;
285	SCNG(script_filtered) = lex_state->script_filtered;
286	SCNG(script_filtered_size) = lex_state->script_filtered_size;
287	SCNG(input_filter) = lex_state->input_filter;
288	SCNG(output_filter) = lex_state->output_filter;
289	SCNG(script_encoding) = lex_state->script_encoding;
290
291	SCNG(on_event) = lex_state->on_event;
292	SCNG(on_event_context) = lex_state->on_event_context;
293
294	CG(ast) = lex_state->ast;
295	CG(ast_arena) = lex_state->ast_arena;
296
297	RESET_DOC_COMMENT();
298}
299
300ZEND_API void zend_destroy_file_handle(zend_file_handle *file_handle)
301{
302	zend_llist_del_element(&CG(open_files), file_handle, (int (*)(void *, void *)) zend_compare_file_handles);
303	/* zend_file_handle_dtor() operates on the copy, so we have to NULLify the original here */
304	file_handle->opened_path = NULL;
305	if (file_handle->free_filename) {
306		file_handle->filename = NULL;
307	}
308}
309
310ZEND_API zend_result zend_lex_tstring(zval *zv, zend_lexer_ident_ref ident_ref)
311{
312	char *ident = (char *) SCNG(yy_start) + ident_ref.offset;
313	size_t length = ident_ref.len;
314	if (length == sizeof("<?=")-1 && memcmp(ident, "<?=", sizeof("<?=")-1) == 0) {
315		zend_throw_exception(zend_ce_parse_error, "Cannot use \"<?=\" as an identifier", 0);
316		return FAILURE;
317	}
318
319	if (SCNG(on_event)) {
320		SCNG(on_event)(ON_FEEDBACK, T_STRING, 0, ident, length, SCNG(on_event_context));
321	}
322
323	ZVAL_STRINGL(zv, ident, length);
324	return SUCCESS;
325}
326
327#define BOM_UTF32_BE	"\x00\x00\xfe\xff"
328#define	BOM_UTF32_LE	"\xff\xfe\x00\x00"
329#define	BOM_UTF16_BE	"\xfe\xff"
330#define	BOM_UTF16_LE	"\xff\xfe"
331#define	BOM_UTF8		"\xef\xbb\xbf"
332
333static const zend_encoding *zend_multibyte_detect_utf_encoding(const unsigned char *script, size_t script_size)
334{
335	const unsigned char *p;
336	int wchar_size = 2;
337	int le = 0;
338
339	/* utf-16 or utf-32? */
340	p = script;
341	assert(p >= script);
342	while ((size_t)(p-script) < script_size) {
343		p = memchr(p, 0, script_size-(p-script)-2);
344		if (!p) {
345			break;
346		}
347		if (*(p+1) == '\0' && *(p+2) == '\0') {
348			wchar_size = 4;
349			break;
350		}
351
352		/* searching for UTF-32 specific byte orders, so this will do */
353		p += 4;
354	}
355
356	/* BE or LE? */
357	p = script;
358	assert(p >= script);
359	while ((size_t)(p-script) < script_size) {
360		if (*p == '\0' && *(p+wchar_size-1) != '\0') {
361			/* BE */
362			le = 0;
363			break;
364		} else if (*p != '\0' && *(p+wchar_size-1) == '\0') {
365			/* LE* */
366			le = 1;
367			break;
368		}
369		p += wchar_size;
370	}
371
372	if (wchar_size == 2) {
373		return le ? zend_multibyte_encoding_utf16le : zend_multibyte_encoding_utf16be;
374	} else {
375		return le ? zend_multibyte_encoding_utf32le : zend_multibyte_encoding_utf32be;
376	}
377
378	return NULL;
379}
380
381static const zend_encoding* zend_multibyte_detect_unicode(void)
382{
383	const zend_encoding *script_encoding = NULL;
384	int bom_size;
385	unsigned char *pos1, *pos2;
386
387	if (LANG_SCNG(script_org_size) < sizeof(BOM_UTF32_LE)-1) {
388		return NULL;
389	}
390
391	/* check out BOM */
392	if (!memcmp(LANG_SCNG(script_org), BOM_UTF32_BE, sizeof(BOM_UTF32_BE)-1)) {
393		script_encoding = zend_multibyte_encoding_utf32be;
394		bom_size = sizeof(BOM_UTF32_BE)-1;
395	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF32_LE, sizeof(BOM_UTF32_LE)-1)) {
396		script_encoding = zend_multibyte_encoding_utf32le;
397		bom_size = sizeof(BOM_UTF32_LE)-1;
398	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF16_BE, sizeof(BOM_UTF16_BE)-1)) {
399		script_encoding = zend_multibyte_encoding_utf16be;
400		bom_size = sizeof(BOM_UTF16_BE)-1;
401	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF16_LE, sizeof(BOM_UTF16_LE)-1)) {
402		script_encoding = zend_multibyte_encoding_utf16le;
403		bom_size = sizeof(BOM_UTF16_LE)-1;
404	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF8, sizeof(BOM_UTF8)-1)) {
405		script_encoding = zend_multibyte_encoding_utf8;
406		bom_size = sizeof(BOM_UTF8)-1;
407	}
408
409	if (script_encoding) {
410		/* remove BOM */
411		LANG_SCNG(script_org) += bom_size;
412		LANG_SCNG(script_org_size) -= bom_size;
413
414		return script_encoding;
415	}
416
417	/* script contains NULL bytes -> auto-detection */
418	if ((pos1 = memchr(LANG_SCNG(script_org), 0, LANG_SCNG(script_org_size)))) {
419		/* check if the NULL byte is after the __HALT_COMPILER(); */
420		pos2 = LANG_SCNG(script_org);
421
422		while ((size_t)(pos1 - pos2) >= sizeof("__HALT_COMPILER();")-1) {
423			pos2 = memchr(pos2, '_', pos1 - pos2);
424			if (!pos2) break;
425			pos2++;
426			if (strncasecmp((char*)pos2, "_HALT_COMPILER", sizeof("_HALT_COMPILER")-1) == 0) {
427				pos2 += sizeof("_HALT_COMPILER")-1;
428				while (*pos2 == ' '  ||
429					   *pos2 == '\t' ||
430					   *pos2 == '\r' ||
431					   *pos2 == '\n') {
432					pos2++;
433				}
434				if (*pos2 == '(') {
435					pos2++;
436					while (*pos2 == ' '  ||
437						   *pos2 == '\t' ||
438						   *pos2 == '\r' ||
439						   *pos2 == '\n') {
440						pos2++;
441					}
442					if (*pos2 == ')') {
443						pos2++;
444						while (*pos2 == ' '  ||
445							   *pos2 == '\t' ||
446							   *pos2 == '\r' ||
447							   *pos2 == '\n') {
448							pos2++;
449						}
450						if (*pos2 == ';') {
451							return NULL;
452						}
453					}
454				}
455			}
456		}
457		/* make best effort if BOM is missing */
458		return zend_multibyte_detect_utf_encoding(LANG_SCNG(script_org), LANG_SCNG(script_org_size));
459	}
460
461	return NULL;
462}
463
464static const zend_encoding* zend_multibyte_find_script_encoding(void)
465{
466	const zend_encoding *script_encoding;
467
468	if (CG(detect_unicode)) {
469		/* check out bom(byte order mark) and see if containing wchars */
470		script_encoding = zend_multibyte_detect_unicode();
471		if (script_encoding != NULL) {
472			/* bom or wchar detection is prior to 'script_encoding' option */
473			return script_encoding;
474		}
475	}
476
477	/* if no script_encoding specified, just leave alone */
478	if (!CG(script_encoding_list) || !CG(script_encoding_list_size)) {
479		return NULL;
480	}
481
482	/* if multiple encodings specified, detect automagically */
483	if (CG(script_encoding_list_size) > 1) {
484		return zend_multibyte_encoding_detector(LANG_SCNG(script_org), LANG_SCNG(script_org_size), CG(script_encoding_list), CG(script_encoding_list_size));
485	}
486
487	return CG(script_encoding_list)[0];
488}
489
490ZEND_API zend_result zend_multibyte_set_filter(const zend_encoding *onetime_encoding)
491{
492	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
493	const zend_encoding *script_encoding = onetime_encoding ? onetime_encoding: zend_multibyte_find_script_encoding();
494
495	if (!script_encoding) {
496		return FAILURE;
497	}
498
499	/* judge input/output filter */
500	LANG_SCNG(script_encoding) = script_encoding;
501	LANG_SCNG(input_filter) = NULL;
502	LANG_SCNG(output_filter) = NULL;
503
504	if (!internal_encoding || LANG_SCNG(script_encoding) == internal_encoding) {
505		if (!zend_multibyte_check_lexer_compatibility(LANG_SCNG(script_encoding))) {
506			/* and if not, work around w/ script_encoding -> utf-8 -> script_encoding conversion */
507			LANG_SCNG(input_filter) = encoding_filter_script_to_intermediate;
508			LANG_SCNG(output_filter) = encoding_filter_intermediate_to_script;
509		} else {
510			LANG_SCNG(input_filter) = NULL;
511			LANG_SCNG(output_filter) = NULL;
512		}
513		return SUCCESS;
514	}
515
516	if (zend_multibyte_check_lexer_compatibility(internal_encoding)) {
517		LANG_SCNG(input_filter) = encoding_filter_script_to_internal;
518		LANG_SCNG(output_filter) = NULL;
519	} else if (zend_multibyte_check_lexer_compatibility(LANG_SCNG(script_encoding))) {
520		LANG_SCNG(input_filter) = NULL;
521		LANG_SCNG(output_filter) = encoding_filter_script_to_internal;
522	} else {
523		/* both script and internal encodings are incompatible w/ flex */
524		LANG_SCNG(input_filter) = encoding_filter_script_to_intermediate;
525		LANG_SCNG(output_filter) = encoding_filter_intermediate_to_internal;
526	}
527
528	return SUCCESS;
529}
530
531ZEND_API zend_result open_file_for_scanning(zend_file_handle *file_handle)
532{
533	char *buf;
534	size_t size;
535	zend_string *compiled_filename;
536
537	if (zend_stream_fixup(file_handle, &buf, &size) == FAILURE) {
538		/* Still add it to open_files to make destroy_file_handle work */
539		zend_llist_add_element(&CG(open_files), file_handle);
540		return FAILURE;
541	}
542
543	ZEND_ASSERT(!EG(exception) && "stream_fixup() should have failed");
544	zend_llist_add_element(&CG(open_files), file_handle);
545	if (file_handle->handle.stream.handle >= (void*)file_handle && file_handle->handle.stream.handle <= (void*)(file_handle+1)) {
546		zend_file_handle *fh = (zend_file_handle*)zend_llist_get_last(&CG(open_files));
547		size_t diff = (char*)file_handle->handle.stream.handle - (char*)file_handle;
548		fh->handle.stream.handle = (void*)(((char*)fh) + diff);
549		file_handle->handle.stream.handle = fh->handle.stream.handle;
550	}
551
552	/* Reset the scanner for scanning the new file */
553	SCNG(yy_in) = file_handle;
554	SCNG(yy_start) = NULL;
555
556	if (size != (size_t)-1) {
557		if (CG(multibyte)) {
558			SCNG(script_org) = (unsigned char*)buf;
559			SCNG(script_org_size) = size;
560			SCNG(script_filtered) = NULL;
561
562			zend_multibyte_set_filter(NULL);
563
564			if (SCNG(input_filter)) {
565				if ((size_t)-1 == SCNG(input_filter)(&SCNG(script_filtered), &SCNG(script_filtered_size), SCNG(script_org), SCNG(script_org_size))) {
566					zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
567							"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
568				}
569				buf = (char*)SCNG(script_filtered);
570				size = SCNG(script_filtered_size);
571			}
572		}
573		SCNG(yy_start) = (unsigned char *)buf;
574		yy_scan_buffer(buf, (unsigned int)size);
575	} else {
576		zend_error_noreturn(E_COMPILE_ERROR, "zend_stream_mmap() failed");
577	}
578
579	if (CG(skip_shebang)) {
580		BEGIN(SHEBANG);
581	} else {
582		BEGIN(INITIAL);
583	}
584
585	if (file_handle->opened_path) {
586		compiled_filename = zend_string_copy(file_handle->opened_path);
587	} else {
588		compiled_filename = zend_string_init(file_handle->filename, strlen(file_handle->filename), 0);
589	}
590
591	zend_set_compiled_filename(compiled_filename);
592	zend_string_release_ex(compiled_filename, 0);
593
594	RESET_DOC_COMMENT();
595	CG(zend_lineno) = 1;
596	CG(increment_lineno) = 0;
597	return SUCCESS;
598}
599END_EXTERN_C()
600
601static zend_op_array *zend_compile(int type)
602{
603	zend_op_array *op_array = NULL;
604	zend_bool original_in_compilation = CG(in_compilation);
605
606	CG(in_compilation) = 1;
607	CG(ast) = NULL;
608	CG(ast_arena) = zend_arena_create(1024 * 32);
609
610	if (!zendparse()) {
611		int last_lineno = CG(zend_lineno);
612		zend_file_context original_file_context;
613		zend_oparray_context original_oparray_context;
614		zend_op_array *original_active_op_array = CG(active_op_array);
615
616		op_array = emalloc(sizeof(zend_op_array));
617		init_op_array(op_array, type, INITIAL_OP_ARRAY_SIZE);
618		CG(active_op_array) = op_array;
619
620		/* Use heap to not waste arena memory */
621		op_array->fn_flags |= ZEND_ACC_HEAP_RT_CACHE;
622
623		if (zend_ast_process) {
624			zend_ast_process(CG(ast));
625		}
626
627		zend_file_context_begin(&original_file_context);
628		zend_oparray_context_begin(&original_oparray_context);
629		zend_compile_top_stmt(CG(ast));
630		CG(zend_lineno) = last_lineno;
631		zend_emit_final_return(type == ZEND_USER_FUNCTION);
632		op_array->line_start = 1;
633		op_array->line_end = last_lineno;
634		pass_two(op_array);
635		zend_oparray_context_end(&original_oparray_context);
636		zend_file_context_end(&original_file_context);
637
638		CG(active_op_array) = original_active_op_array;
639	}
640
641	zend_ast_destroy(CG(ast));
642	zend_arena_destroy(CG(ast_arena));
643
644	CG(in_compilation) = original_in_compilation;
645
646	return op_array;
647}
648
649ZEND_API zend_op_array *compile_file(zend_file_handle *file_handle, int type)
650{
651	zend_lex_state original_lex_state;
652	zend_op_array *op_array = NULL;
653	zend_save_lexical_state(&original_lex_state);
654
655	if (open_file_for_scanning(file_handle)==FAILURE) {
656		if (!EG(exception)) {
657			if (type==ZEND_REQUIRE) {
658				zend_message_dispatcher(ZMSG_FAILED_REQUIRE_FOPEN, file_handle->filename);
659			} else {
660				zend_message_dispatcher(ZMSG_FAILED_INCLUDE_FOPEN, file_handle->filename);
661			}
662		}
663	} else {
664		op_array = zend_compile(ZEND_USER_FUNCTION);
665	}
666
667	zend_restore_lexical_state(&original_lex_state);
668	return op_array;
669}
670
671ZEND_API zend_ast *zend_compile_string_to_ast(
672		zend_string *code, zend_arena **ast_arena, const char *filename) {
673	zval code_zv;
674	zend_bool original_in_compilation;
675	zend_lex_state original_lex_state;
676	zend_ast *ast;
677
678	ZVAL_STR_COPY(&code_zv, code);
679
680	original_in_compilation = CG(in_compilation);
681	CG(in_compilation) = 1;
682
683	zend_save_lexical_state(&original_lex_state);
684	zend_prepare_string_for_scanning(&code_zv, filename);
685	CG(ast) = NULL;
686	CG(ast_arena) = zend_arena_create(1024 * 32);
687	LANG_SCNG(yy_state) = yycINITIAL;
688
689	if (zendparse() != 0) {
690		zend_ast_destroy(CG(ast));
691		zend_arena_destroy(CG(ast_arena));
692		CG(ast) = NULL;
693	}
694
695	/* restore_lexical_state changes CG(ast) and CG(ast_arena) */
696	ast = CG(ast);
697	*ast_arena = CG(ast_arena);
698
699	zend_restore_lexical_state(&original_lex_state);
700	CG(in_compilation) = original_in_compilation;
701
702	zval_ptr_dtor_str(&code_zv);
703
704	return ast;
705}
706
707zend_op_array *compile_filename(int type, zval *filename)
708{
709	zend_file_handle file_handle;
710	zval tmp;
711	zend_op_array *retval;
712	zend_string *opened_path = NULL;
713
714	if (Z_TYPE_P(filename) != IS_STRING) {
715		ZVAL_STR(&tmp, zval_get_string(filename));
716		filename = &tmp;
717	}
718	zend_stream_init_filename(&file_handle, Z_STRVAL_P(filename));
719
720	retval = zend_compile_file(&file_handle, type);
721	if (retval && file_handle.handle.stream.handle) {
722		if (!file_handle.opened_path) {
723			file_handle.opened_path = opened_path = zend_string_copy(Z_STR_P(filename));
724		}
725
726		zend_hash_add_empty_element(&EG(included_files), file_handle.opened_path);
727
728		if (opened_path) {
729			zend_string_release_ex(opened_path, 0);
730		}
731	}
732	zend_destroy_file_handle(&file_handle);
733
734	if (UNEXPECTED(filename == &tmp)) {
735		zval_ptr_dtor(&tmp);
736	}
737	return retval;
738}
739
740ZEND_API void zend_prepare_string_for_scanning(zval *str, const char *filename)
741{
742	char *buf;
743	size_t size, old_len;
744	zend_string *new_compiled_filename;
745
746	/* enforce ZEND_MMAP_AHEAD trailing NULLs for flex... */
747	old_len = Z_STRLEN_P(str);
748	Z_STR_P(str) = zend_string_extend(Z_STR_P(str), old_len + ZEND_MMAP_AHEAD, 0);
749	Z_TYPE_INFO_P(str) = IS_STRING_EX;
750	memset(Z_STRVAL_P(str) + old_len, 0, ZEND_MMAP_AHEAD + 1);
751
752	SCNG(yy_in) = NULL;
753	SCNG(yy_start) = NULL;
754
755	buf = Z_STRVAL_P(str);
756	size = old_len;
757
758	if (CG(multibyte)) {
759		SCNG(script_org) = (unsigned char*)buf;
760		SCNG(script_org_size) = size;
761		SCNG(script_filtered) = NULL;
762
763		zend_multibyte_set_filter(zend_multibyte_get_internal_encoding());
764
765		if (SCNG(input_filter)) {
766			if ((size_t)-1 == SCNG(input_filter)(&SCNG(script_filtered), &SCNG(script_filtered_size), SCNG(script_org), SCNG(script_org_size))) {
767				zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
768						"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
769			}
770			buf = (char*)SCNG(script_filtered);
771			size = SCNG(script_filtered_size);
772		}
773	}
774
775	yy_scan_buffer(buf, (unsigned int)size);
776
777	new_compiled_filename = zend_string_init(filename, strlen(filename), 0);
778	zend_set_compiled_filename(new_compiled_filename);
779	zend_string_release_ex(new_compiled_filename, 0);
780	CG(zend_lineno) = 1;
781	CG(increment_lineno) = 0;
782	RESET_DOC_COMMENT();
783}
784
785
786ZEND_API size_t zend_get_scanned_file_offset(void)
787{
788	size_t offset = SCNG(yy_cursor) - SCNG(yy_start);
789	if (SCNG(input_filter)) {
790		size_t original_offset = offset, length = 0;
791		do {
792			unsigned char *p = NULL;
793			if ((size_t)-1 == SCNG(input_filter)(&p, &length, SCNG(script_org), offset)) {
794				return (size_t)-1;
795			}
796			efree(p);
797			if (length > original_offset) {
798				offset--;
799			} else if (length < original_offset) {
800				offset++;
801			}
802		} while (original_offset != length);
803	}
804	return offset;
805}
806
807zend_op_array *compile_string(zend_string *source_string, const char *filename)
808{
809	zend_lex_state original_lex_state;
810	zend_op_array *op_array = NULL;
811	zval tmp;
812
813	if (ZSTR_LEN(source_string) == 0) {
814		return NULL;
815	}
816
817	ZVAL_STR_COPY(&tmp, source_string);
818
819	zend_save_lexical_state(&original_lex_state);
820	zend_prepare_string_for_scanning(&tmp, filename);
821	BEGIN(ST_IN_SCRIPTING);
822	op_array = zend_compile(ZEND_EVAL_CODE);
823
824	zend_restore_lexical_state(&original_lex_state);
825	zval_ptr_dtor(&tmp);
826
827	return op_array;
828}
829
830
831BEGIN_EXTERN_C()
832zend_result highlight_file(const char *filename, zend_syntax_highlighter_ini *syntax_highlighter_ini)
833{
834	zend_lex_state original_lex_state;
835	zend_file_handle file_handle;
836
837	zend_stream_init_filename(&file_handle, filename);
838	zend_save_lexical_state(&original_lex_state);
839	if (open_file_for_scanning(&file_handle)==FAILURE) {
840		zend_message_dispatcher(ZMSG_FAILED_HIGHLIGHT_FOPEN, filename);
841		zend_restore_lexical_state(&original_lex_state);
842		return FAILURE;
843	}
844	zend_highlight(syntax_highlighter_ini);
845	if (SCNG(script_filtered)) {
846		efree(SCNG(script_filtered));
847		SCNG(script_filtered) = NULL;
848	}
849	zend_destroy_file_handle(&file_handle);
850	zend_restore_lexical_state(&original_lex_state);
851	return SUCCESS;
852}
853
854void highlight_string(zval *str, zend_syntax_highlighter_ini *syntax_highlighter_ini, const char *str_name)
855{
856	zend_lex_state original_lex_state;
857	zval tmp;
858
859	if (UNEXPECTED(Z_TYPE_P(str) != IS_STRING)) {
860		ZVAL_STR(&tmp, zval_get_string_func(str));
861		str = &tmp;
862	}
863	zend_save_lexical_state(&original_lex_state);
864	zend_prepare_string_for_scanning(str, str_name);
865	BEGIN(INITIAL);
866	zend_highlight(syntax_highlighter_ini);
867	if (SCNG(script_filtered)) {
868		efree(SCNG(script_filtered));
869		SCNG(script_filtered) = NULL;
870	}
871	zend_restore_lexical_state(&original_lex_state);
872	if (UNEXPECTED(str == &tmp)) {
873		zval_ptr_dtor(&tmp);
874	}
875}
876
877ZEND_API void zend_multibyte_yyinput_again(zend_encoding_filter old_input_filter, const zend_encoding *old_encoding)
878{
879	size_t length;
880	unsigned char *new_yy_start;
881
882	/* convert and set */
883	if (!SCNG(input_filter)) {
884		if (SCNG(script_filtered)) {
885			efree(SCNG(script_filtered));
886			SCNG(script_filtered) = NULL;
887		}
888		SCNG(script_filtered_size) = 0;
889		length = SCNG(script_org_size);
890		new_yy_start = SCNG(script_org);
891	} else {
892		if ((size_t)-1 == SCNG(input_filter)(&new_yy_start, &length, SCNG(script_org), SCNG(script_org_size))) {
893			zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
894					"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
895		}
896		if (SCNG(script_filtered)) {
897			efree(SCNG(script_filtered));
898		}
899		SCNG(script_filtered) = new_yy_start;
900		SCNG(script_filtered_size) = length;
901	}
902
903	SCNG(yy_cursor) = new_yy_start + (SCNG(yy_cursor) - SCNG(yy_start));
904	SCNG(yy_marker) = new_yy_start + (SCNG(yy_marker) - SCNG(yy_start));
905	SCNG(yy_text) = new_yy_start + (SCNG(yy_text) - SCNG(yy_start));
906	SCNG(yy_limit) = new_yy_start + length;
907
908	SCNG(yy_start) = new_yy_start;
909}
910
911
912// TODO: avoid reallocation ???
913# define zend_copy_value(zendlval, yytext, yyleng) \
914	if (SCNG(output_filter)) { \
915		size_t sz = 0; \
916		char *s = NULL; \
917		SCNG(output_filter)((unsigned char **)&s, &sz, (unsigned char *)yytext, (size_t)yyleng); \
918		ZVAL_STRINGL(zendlval, s, sz); \
919		efree(s); \
920	} else if (yyleng == 1) { \
921		ZVAL_INTERNED_STR(zendlval, ZSTR_CHAR((zend_uchar)*(yytext))); \
922	} else { \
923		ZVAL_STRINGL(zendlval, yytext, yyleng); \
924	}
925
926static zend_result zend_scan_escape_string(zval *zendlval, char *str, int len, char quote_type)
927{
928	register char *s, *t;
929	char *end;
930
931	if (len <= 1) {
932		if (len < 1) {
933			ZVAL_EMPTY_STRING(zendlval);
934		} else {
935			zend_uchar c = (zend_uchar)*str;
936			if (c == '\n' || c == '\r') {
937				CG(zend_lineno)++;
938			}
939			ZVAL_INTERNED_STR(zendlval, ZSTR_CHAR(c));
940		}
941		goto skip_escape_conversion;
942	}
943
944	ZVAL_STRINGL(zendlval, str, len);
945
946	/* convert escape sequences */
947	s = Z_STRVAL_P(zendlval);
948	end = s+Z_STRLEN_P(zendlval);
949	while (1) {
950		if (UNEXPECTED(*s=='\\')) {
951			break;
952		}
953		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
954			CG(zend_lineno)++;
955		}
956		s++;
957		if (s == end) {
958			goto skip_escape_conversion;
959		}
960	}
961
962	t = s;
963	while (s<end) {
964		if (*s=='\\') {
965			s++;
966			if (s >= end) {
967				*t++ = '\\';
968				break;
969			}
970
971			switch(*s) {
972				case 'n':
973					*t++ = '\n';
974					break;
975				case 'r':
976					*t++ = '\r';
977					break;
978				case 't':
979					*t++ = '\t';
980					break;
981				case 'f':
982					*t++ = '\f';
983					break;
984				case 'v':
985					*t++ = '\v';
986					break;
987				case 'e':
988#ifdef ZEND_WIN32
989					*t++ = VK_ESCAPE;
990#else
991					*t++ = '\e';
992#endif
993					break;
994				case '"':
995				case '`':
996					if (*s != quote_type) {
997						*t++ = '\\';
998						*t++ = *s;
999						break;
1000					}
1001				case '\\':
1002				case '$':
1003					*t++ = *s;
1004					break;
1005				case 'x':
1006				case 'X':
1007					if (ZEND_IS_HEX(*(s+1))) {
1008						char hex_buf[3] = { 0, 0, 0 };
1009
1010						hex_buf[0] = *(++s);
1011						if (ZEND_IS_HEX(*(s+1))) {
1012							hex_buf[1] = *(++s);
1013						}
1014						*t++ = (char) ZEND_STRTOL(hex_buf, NULL, 16);
1015					} else {
1016						*t++ = '\\';
1017						*t++ = *s;
1018					}
1019					break;
1020				/* UTF-8 codepoint escape, format: /\\u\{\x+\}/ */
1021				case 'u':
1022					{
1023						/* cache where we started so we can parse after validating */
1024						char *start = s + 1;
1025						size_t len = 0;
1026						zend_bool valid = 1;
1027						unsigned long codepoint;
1028
1029						if (*start != '{') {
1030							/* we silently let this pass to avoid breaking code
1031							 * with JSON in string literals (e.g. "\"\u202e\""
1032							 */
1033							*t++ = '\\';
1034							*t++ = 'u';
1035							break;
1036						} else {
1037							/* on the other hand, invalid \u{blah} errors */
1038							s++;
1039							len++;
1040							s++;
1041							while (*s != '}') {
1042								if (!ZEND_IS_HEX(*s)) {
1043									valid = 0;
1044									break;
1045								} else {
1046									len++;
1047								}
1048								s++;
1049							}
1050							if (*s == '}') {
1051								valid = 1;
1052								len++;
1053							}
1054						}
1055
1056						/* \u{} is invalid */
1057						if (len <= 2) {
1058							valid = 0;
1059						}
1060
1061						if (!valid) {
1062							zend_throw_exception(zend_ce_parse_error,
1063								"Invalid UTF-8 codepoint escape sequence", 0);
1064							zval_ptr_dtor(zendlval);
1065							ZVAL_UNDEF(zendlval);
1066							return FAILURE;
1067						}
1068
1069						errno = 0;
1070						codepoint = strtoul(start + 1, NULL, 16);
1071
1072						/* per RFC 3629, UTF-8 can only represent 21 bits */
1073						if (codepoint > 0x10FFFF || errno) {
1074							zend_throw_exception(zend_ce_parse_error,
1075								"Invalid UTF-8 codepoint escape sequence: Codepoint too large", 0);
1076							zval_ptr_dtor(zendlval);
1077							ZVAL_UNDEF(zendlval);
1078							return FAILURE;
1079						}
1080
1081						/* based on https://en.wikipedia.org/wiki/UTF-8#Sample_code */
1082						if (codepoint < 0x80) {
1083							*t++ = codepoint;
1084						} else if (codepoint <= 0x7FF) {
1085							*t++ = (codepoint >> 6) + 0xC0;
1086							*t++ = (codepoint & 0x3F) + 0x80;
1087						} else if (codepoint <= 0xFFFF) {
1088							*t++ = (codepoint >> 12) + 0xE0;
1089							*t++ = ((codepoint >> 6) & 0x3F) + 0x80;
1090							*t++ = (codepoint & 0x3F) + 0x80;
1091						} else if (codepoint <= 0x10FFFF) {
1092							*t++ = (codepoint >> 18) + 0xF0;
1093							*t++ = ((codepoint >> 12) & 0x3F) + 0x80;
1094							*t++ = ((codepoint >> 6) & 0x3F) + 0x80;
1095							*t++ = (codepoint & 0x3F) + 0x80;
1096						}
1097					}
1098					break;
1099				default:
1100					/* check for an octal */
1101					if (ZEND_IS_OCT(*s)) {
1102						char octal_buf[4] = { 0, 0, 0, 0 };
1103
1104						octal_buf[0] = *s;
1105						if (ZEND_IS_OCT(*(s+1))) {
1106							octal_buf[1] = *(++s);
1107							if (ZEND_IS_OCT(*(s+1))) {
1108								octal_buf[2] = *(++s);
1109							}
1110						}
1111						if (octal_buf[2] && (octal_buf[0] > '3') && !SCNG(heredoc_scan_ahead)) {
1112							/* 3 octit values must not overflow 0xFF (\377) */
1113							zend_error(E_COMPILE_WARNING, "Octal escape sequence overflow \\%s is greater than \\377", octal_buf);
1114						}
1115
1116						*t++ = (char) ZEND_STRTOL(octal_buf, NULL, 8);
1117					} else {
1118						*t++ = '\\';
1119						*t++ = *s;
1120					}
1121					break;
1122			}
1123		} else {
1124			*t++ = *s;
1125		}
1126
1127		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
1128			CG(zend_lineno)++;
1129		}
1130		s++;
1131	}
1132	*t = 0;
1133	Z_STRLEN_P(zendlval) = t - Z_STRVAL_P(zendlval);
1134
1135skip_escape_conversion:
1136	if (SCNG(output_filter)) {
1137		size_t sz = 0;
1138		unsigned char *str;
1139		// TODO: avoid realocation ???
1140		s = Z_STRVAL_P(zendlval);
1141		SCNG(output_filter)(&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
1142		zval_ptr_dtor(zendlval);
1143		ZVAL_STRINGL(zendlval, (char *) str, sz);
1144		efree(str);
1145	}
1146	return SUCCESS;
1147}
1148
1149#define HEREDOC_USING_SPACES 1
1150#define HEREDOC_USING_TABS 2
1151
1152static const char *next_newline(const char *str, const char *end, size_t *newline_len) {
1153	for (; str < end; str++) {
1154		if (*str == '\r') {
1155			*newline_len = str + 1 < end && *(str + 1) == '\n' ? 2 : 1;
1156			return str;
1157		} else if (*str == '\n') {
1158			*newline_len = 1;
1159			return str;
1160		}
1161	}
1162	*newline_len = 0;
1163	return NULL;
1164}
1165
1166static zend_bool strip_multiline_string_indentation(
1167	zval *zendlval, int indentation, zend_bool using_spaces,
1168	zend_bool newline_at_start, zend_bool newline_at_end)
1169{
1170	const char *str = Z_STRVAL_P(zendlval), *end = str + Z_STRLEN_P(zendlval);
1171	char *copy = Z_STRVAL_P(zendlval);
1172
1173	int newline_count = 0;
1174	size_t newline_len;
1175	const char *nl;
1176
1177	if (!newline_at_start) {
1178		nl = next_newline(str, end, &newline_len);
1179		if (!nl) {
1180			return 1;
1181		}
1182
1183		str = nl + newline_len;
1184		copy = (char *) nl + newline_len;
1185		newline_count++;
1186	} else {
1187		nl = str;
1188	}
1189
1190	/* <= intentional */
1191	while (str <= end && nl) {
1192		size_t skip;
1193		nl = next_newline(str, end, &newline_len);
1194		if (!nl && newline_at_end) {
1195			nl = end;
1196		}
1197
1198		/* Try to skip indentation */
1199		for (skip = 0; skip < indentation; skip++, str++) {
1200			if (str == nl) {
1201				/* Don't require full indentation on whitespace-only lines */
1202				break;
1203			}
1204
1205			if (str == end || (*str != ' ' && *str != '\t')) {
1206				CG(zend_lineno) += newline_count;
1207				zend_throw_exception_ex(zend_ce_parse_error, 0,
1208					"Invalid body indentation level (expecting an indentation level of at least %d)", indentation);
1209				goto error;
1210			}
1211
1212			if ((!using_spaces && *str == ' ') || (using_spaces && *str == '\t')) {
1213				CG(zend_lineno) += newline_count;
1214				zend_throw_exception(zend_ce_parse_error,
1215					"Invalid indentation - tabs and spaces cannot be mixed", 0);
1216				goto error;
1217			}
1218		}
1219
1220		if (str == end) {
1221			break;
1222		}
1223
1224		size_t len = nl ? (nl - str + newline_len) : (end - str);
1225		memmove(copy, str, len);
1226		str += len;
1227		copy += len;
1228		newline_count++;
1229	}
1230
1231	*copy = '\0';
1232	Z_STRLEN_P(zendlval) = copy - Z_STRVAL_P(zendlval);
1233	return 1;
1234
1235error:
1236	zval_ptr_dtor_str(zendlval);
1237	ZVAL_UNDEF(zendlval);
1238
1239	return 0;
1240}
1241
1242static void copy_heredoc_label_stack(void *void_heredoc_label)
1243{
1244	zend_heredoc_label *heredoc_label = void_heredoc_label;
1245	zend_heredoc_label *new_heredoc_label = emalloc(sizeof(zend_heredoc_label));
1246
1247	*new_heredoc_label = *heredoc_label;
1248	new_heredoc_label->label = estrndup(heredoc_label->label, heredoc_label->length);
1249
1250	zend_ptr_stack_push(&SCNG(heredoc_label_stack), (void *) new_heredoc_label);
1251}
1252
1253/* Check that { }, [ ], ( ) are nested correctly */
1254static void report_bad_nesting(char opening, int opening_lineno, char closing)
1255{
1256	char   buf[256];
1257	size_t used = 0;
1258
1259	used = snprintf(buf, sizeof(buf), "Unclosed '%c'", opening);
1260
1261	if (opening_lineno != CG(zend_lineno)) {
1262		used += snprintf(buf + used, sizeof(buf) - used, " on line %d", opening_lineno);
1263	}
1264
1265	if (closing) { 	/* 'closing' will be 0 if at end of file */
1266		used += snprintf(buf + used, sizeof(buf) - used, " does not match '%c'", closing);
1267	}
1268
1269	zend_throw_exception(zend_ce_parse_error, buf, 0);
1270}
1271
1272static void enter_nesting(char opening)
1273{
1274	zend_nest_location nest_loc = {opening, CG(zend_lineno)};
1275	zend_stack_push(&SCNG(nest_location_stack), &nest_loc);
1276}
1277
1278static zend_result exit_nesting(char closing)
1279{
1280	if (zend_stack_is_empty(&SCNG(nest_location_stack))) {
1281		zend_throw_exception_ex(zend_ce_parse_error, 0, "Unmatched '%c'", closing);
1282		return FAILURE;
1283	}
1284
1285	zend_nest_location *nest_loc = zend_stack_top(&SCNG(nest_location_stack));
1286	char opening = nest_loc->text;
1287
1288	if ((opening == '{' && closing != '}') ||
1289	    (opening == '[' && closing != ']') ||
1290	    (opening == '(' && closing != ')')) {
1291		report_bad_nesting(opening, nest_loc->lineno, closing);
1292		return FAILURE;
1293	}
1294
1295	zend_stack_del_top(&SCNG(nest_location_stack));
1296	return SUCCESS;
1297}
1298
1299static zend_result check_nesting_at_end()
1300{
1301	if (!zend_stack_is_empty(&SCNG(nest_location_stack))) {
1302		zend_nest_location *nest_loc = zend_stack_top(&SCNG(nest_location_stack));
1303		report_bad_nesting(nest_loc->text, nest_loc->lineno, 0);
1304		return FAILURE;
1305	}
1306
1307	return SUCCESS;
1308}
1309
1310#define PARSER_MODE() \
1311	EXPECTED(elem != NULL)
1312
1313#define RETURN_TOKEN(_token) do { \
1314		token = _token; \
1315		goto emit_token; \
1316	} while (0)
1317
1318#define RETURN_TOKEN_WITH_VAL(_token) do { \
1319		token = _token; \
1320		goto emit_token_with_val; \
1321	} while (0)
1322
1323#define RETURN_TOKEN_WITH_STR(_token, _offset) do { \
1324		token = _token; \
1325		offset = _offset; \
1326		goto emit_token_with_str; \
1327	} while (0)
1328
1329#define RETURN_TOKEN_WITH_IDENT(_token) do { \
1330		token = _token; \
1331		goto emit_token_with_ident; \
1332	} while (0)
1333
1334#define RETURN_OR_SKIP_TOKEN(_token) do { \
1335		token = _token; \
1336		if (PARSER_MODE()) { \
1337			goto skip_token; \
1338		} \
1339		goto emit_token; \
1340	} while (0)
1341
1342#define RETURN_EXIT_NESTING_TOKEN(_token) do { \
1343		if (exit_nesting(_token) && PARSER_MODE()) { \
1344			RETURN_TOKEN(T_ERROR); \
1345		} else { \
1346			RETURN_TOKEN(_token); \
1347		} \
1348	} while(0)
1349
1350#define RETURN_END_TOKEN do { \
1351		if (check_nesting_at_end() && PARSER_MODE()) { \
1352			RETURN_TOKEN(T_ERROR); \
1353		} else { \
1354			RETURN_TOKEN(END); \
1355		} \
1356	} while (0)
1357
1358int ZEND_FASTCALL lex_scan(zval *zendlval, zend_parser_stack_elem *elem)
1359{
1360int token;
1361int offset;
1362int start_line = CG(zend_lineno);
1363
1364	ZVAL_UNDEF(zendlval);
1365restart:
1366	SCNG(yy_text) = YYCURSOR;
1367
1368/*!re2c
1369re2c:yyfill:check = 0;
1370LNUM	[0-9]+(_[0-9]+)*
1371DNUM	({LNUM}?"."{LNUM})|({LNUM}"."{LNUM}?)
1372EXPONENT_DNUM	(({LNUM}|{DNUM})[eE][+-]?{LNUM})
1373HNUM	"0x"[0-9a-fA-F]+(_[0-9a-fA-F]+)*
1374BNUM	"0b"[01]+(_[01]+)*
1375LABEL	[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*
1376WHITESPACE [ \n\r\t]+
1377TABS_AND_SPACES [ \t]*
1378TOKENS [;:,.|^&+-/*=%!~$<>?@]
1379ANY_CHAR [^]
1380NEWLINE ("\r"|"\n"|"\r\n")
1381
1382/* compute yyleng before each rule */
1383<!*> := yyleng = YYCURSOR - SCNG(yy_text);
1384
1385<ST_IN_SCRIPTING>"exit" {
1386	RETURN_TOKEN_WITH_IDENT(T_EXIT);
1387}
1388
1389<ST_IN_SCRIPTING>"die" {
1390	RETURN_TOKEN_WITH_IDENT(T_EXIT);
1391}
1392
1393<ST_IN_SCRIPTING>"fn" {
1394	RETURN_TOKEN_WITH_IDENT(T_FN);
1395}
1396
1397<ST_IN_SCRIPTING>"function" {
1398	RETURN_TOKEN_WITH_IDENT(T_FUNCTION);
1399}
1400
1401<ST_IN_SCRIPTING>"const" {
1402	RETURN_TOKEN_WITH_IDENT(T_CONST);
1403}
1404
1405<ST_IN_SCRIPTING>"return" {
1406	RETURN_TOKEN_WITH_IDENT(T_RETURN);
1407}
1408
1409<ST_IN_SCRIPTING>"#[" {
1410	enter_nesting('[');
1411	RETURN_TOKEN(T_ATTRIBUTE);
1412}
1413
1414<ST_IN_SCRIPTING>"yield"{WHITESPACE}"from"[^a-zA-Z0-9_\x80-\xff] {
1415	yyless(yyleng - 1);
1416	HANDLE_NEWLINES(yytext, yyleng);
1417	RETURN_TOKEN_WITH_IDENT(T_YIELD_FROM);
1418}
1419
1420<ST_IN_SCRIPTING>"yield" {
1421	RETURN_TOKEN_WITH_IDENT(T_YIELD);
1422}
1423
1424<ST_IN_SCRIPTING>"try" {
1425	RETURN_TOKEN_WITH_IDENT(T_TRY);
1426}
1427
1428<ST_IN_SCRIPTING>"catch" {
1429	RETURN_TOKEN_WITH_IDENT(T_CATCH);
1430}
1431
1432<ST_IN_SCRIPTING>"finally" {
1433	RETURN_TOKEN_WITH_IDENT(T_FINALLY);
1434}
1435
1436<ST_IN_SCRIPTING>"throw" {
1437	RETURN_TOKEN_WITH_IDENT(T_THROW);
1438}
1439
1440<ST_IN_SCRIPTING>"if" {
1441	RETURN_TOKEN_WITH_IDENT(T_IF);
1442}
1443
1444<ST_IN_SCRIPTING>"elseif" {
1445	RETURN_TOKEN_WITH_IDENT(T_ELSEIF);
1446}
1447
1448<ST_IN_SCRIPTING>"endif" {
1449	RETURN_TOKEN_WITH_IDENT(T_ENDIF);
1450}
1451
1452<ST_IN_SCRIPTING>"else" {
1453	RETURN_TOKEN_WITH_IDENT(T_ELSE);
1454}
1455
1456<ST_IN_SCRIPTING>"while" {
1457	RETURN_TOKEN_WITH_IDENT(T_WHILE);
1458}
1459
1460<ST_IN_SCRIPTING>"endwhile" {
1461	RETURN_TOKEN_WITH_IDENT(T_ENDWHILE);
1462}
1463
1464<ST_IN_SCRIPTING>"do" {
1465	RETURN_TOKEN_WITH_IDENT(T_DO);
1466}
1467
1468<ST_IN_SCRIPTING>"for" {
1469	RETURN_TOKEN_WITH_IDENT(T_FOR);
1470}
1471
1472<ST_IN_SCRIPTING>"endfor" {
1473	RETURN_TOKEN_WITH_IDENT(T_ENDFOR);
1474}
1475
1476<ST_IN_SCRIPTING>"foreach" {
1477	RETURN_TOKEN_WITH_IDENT(T_FOREACH);
1478}
1479
1480<ST_IN_SCRIPTING>"endforeach" {
1481	RETURN_TOKEN_WITH_IDENT(T_ENDFOREACH);
1482}
1483
1484<ST_IN_SCRIPTING>"declare" {
1485	RETURN_TOKEN_WITH_IDENT(T_DECLARE);
1486}
1487
1488<ST_IN_SCRIPTING>"enddeclare" {
1489	RETURN_TOKEN_WITH_IDENT(T_ENDDECLARE);
1490}
1491
1492<ST_IN_SCRIPTING>"instanceof" {
1493	RETURN_TOKEN_WITH_IDENT(T_INSTANCEOF);
1494}
1495
1496<ST_IN_SCRIPTING>"as" {
1497	RETURN_TOKEN_WITH_IDENT(T_AS);
1498}
1499
1500<ST_IN_SCRIPTING>"switch" {
1501	RETURN_TOKEN_WITH_IDENT(T_SWITCH);
1502}
1503
1504<ST_IN_SCRIPTING>"match" {
1505	RETURN_TOKEN_WITH_IDENT(T_MATCH);
1506}
1507
1508<ST_IN_SCRIPTING>"endswitch" {
1509	RETURN_TOKEN_WITH_IDENT(T_ENDSWITCH);
1510}
1511
1512<ST_IN_SCRIPTING>"case" {
1513	RETURN_TOKEN_WITH_IDENT(T_CASE);
1514}
1515
1516<ST_IN_SCRIPTING>"default" {
1517	RETURN_TOKEN_WITH_IDENT(T_DEFAULT);
1518}
1519
1520<ST_IN_SCRIPTING>"break" {
1521	RETURN_TOKEN_WITH_IDENT(T_BREAK);
1522}
1523
1524<ST_IN_SCRIPTING>"continue" {
1525	RETURN_TOKEN_WITH_IDENT(T_CONTINUE);
1526}
1527
1528<ST_IN_SCRIPTING>"goto" {
1529	RETURN_TOKEN_WITH_IDENT(T_GOTO);
1530}
1531
1532<ST_IN_SCRIPTING>"echo" {
1533	RETURN_TOKEN_WITH_IDENT(T_ECHO);
1534}
1535
1536<ST_IN_SCRIPTING>"print" {
1537	RETURN_TOKEN_WITH_IDENT(T_PRINT);
1538}
1539
1540<ST_IN_SCRIPTING>"class" {
1541	RETURN_TOKEN_WITH_IDENT(T_CLASS);
1542}
1543
1544<ST_IN_SCRIPTING>"interface" {
1545	RETURN_TOKEN_WITH_IDENT(T_INTERFACE);
1546}
1547
1548<ST_IN_SCRIPTING>"trait" {
1549	RETURN_TOKEN_WITH_IDENT(T_TRAIT);
1550}
1551
1552<ST_IN_SCRIPTING>"extends" {
1553	RETURN_TOKEN_WITH_IDENT(T_EXTENDS);
1554}
1555
1556<ST_IN_SCRIPTING>"implements" {
1557	RETURN_TOKEN_WITH_IDENT(T_IMPLEMENTS);
1558}
1559
1560<ST_IN_SCRIPTING>"->" {
1561	yy_push_state(ST_LOOKING_FOR_PROPERTY);
1562	RETURN_TOKEN(T_OBJECT_OPERATOR);
1563}
1564
1565<ST_IN_SCRIPTING>"?->" {
1566	yy_push_state(ST_LOOKING_FOR_PROPERTY);
1567	return T_NULLSAFE_OBJECT_OPERATOR;
1568}
1569
1570<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY>{WHITESPACE}+ {
1571	goto return_whitespace;
1572}
1573
1574<ST_LOOKING_FOR_PROPERTY>"->" {
1575	RETURN_TOKEN(T_OBJECT_OPERATOR);
1576}
1577
1578<ST_LOOKING_FOR_PROPERTY>"?->" {
1579	RETURN_TOKEN(T_NULLSAFE_OBJECT_OPERATOR);
1580}
1581
1582<ST_LOOKING_FOR_PROPERTY>{LABEL} {
1583	yy_pop_state();
1584	RETURN_TOKEN_WITH_STR(T_STRING, 0);
1585}
1586
1587<ST_LOOKING_FOR_PROPERTY>{ANY_CHAR} {
1588	yyless(0);
1589	yy_pop_state();
1590	goto restart;
1591}
1592
1593<ST_IN_SCRIPTING>"::" {
1594	RETURN_TOKEN(T_PAAMAYIM_NEKUDOTAYIM);
1595}
1596
1597<ST_IN_SCRIPTING>"..." {
1598	RETURN_TOKEN(T_ELLIPSIS);
1599}
1600
1601<ST_IN_SCRIPTING>"??" {
1602	RETURN_TOKEN(T_COALESCE);
1603}
1604
1605<ST_IN_SCRIPTING>"new" {
1606	RETURN_TOKEN_WITH_IDENT(T_NEW);
1607}
1608
1609<ST_IN_SCRIPTING>"clone" {
1610	RETURN_TOKEN_WITH_IDENT(T_CLONE);
1611}
1612
1613<ST_IN_SCRIPTING>"var" {
1614	RETURN_TOKEN_WITH_IDENT(T_VAR);
1615}
1616
1617<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("int"|"integer"){TABS_AND_SPACES}")" {
1618	RETURN_TOKEN(T_INT_CAST);
1619}
1620
1621<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("double"|"float"){TABS_AND_SPACES}")" {
1622	RETURN_TOKEN(T_DOUBLE_CAST);
1623}
1624
1625<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"real"{TABS_AND_SPACES}")" {
1626	if (PARSER_MODE()) {
1627		zend_throw_exception(zend_ce_parse_error, "The (real) cast has been removed, use (float) instead", 0);
1628		RETURN_TOKEN(T_ERROR);
1629	}
1630	RETURN_TOKEN(T_DOUBLE_CAST);
1631}
1632
1633<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("string"|"binary"){TABS_AND_SPACES}")" {
1634	RETURN_TOKEN(T_STRING_CAST);
1635}
1636
1637<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"array"{TABS_AND_SPACES}")" {
1638	RETURN_TOKEN(T_ARRAY_CAST);
1639}
1640
1641<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"object"{TABS_AND_SPACES}")" {
1642	RETURN_TOKEN(T_OBJECT_CAST);
1643}
1644
1645<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("bool"|"boolean"){TABS_AND_SPACES}")" {
1646	RETURN_TOKEN(T_BOOL_CAST);
1647}
1648
1649<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("unset"){TABS_AND_SPACES}")" {
1650	RETURN_TOKEN(T_UNSET_CAST);
1651}
1652
1653<ST_IN_SCRIPTING>"eval" {
1654	RETURN_TOKEN_WITH_IDENT(T_EVAL);
1655}
1656
1657<ST_IN_SCRIPTING>"include" {
1658	RETURN_TOKEN_WITH_IDENT(T_INCLUDE);
1659}
1660
1661<ST_IN_SCRIPTING>"include_once" {
1662	RETURN_TOKEN_WITH_IDENT(T_INCLUDE_ONCE);
1663}
1664
1665<ST_IN_SCRIPTING>"require" {
1666	RETURN_TOKEN_WITH_IDENT(T_REQUIRE);
1667}
1668
1669<ST_IN_SCRIPTING>"require_once" {
1670	RETURN_TOKEN_WITH_IDENT(T_REQUIRE_ONCE);
1671}
1672
1673<ST_IN_SCRIPTING>"namespace" {
1674	RETURN_TOKEN_WITH_IDENT(T_NAMESPACE);
1675}
1676
1677<ST_IN_SCRIPTING>"use" {
1678	RETURN_TOKEN_WITH_IDENT(T_USE);
1679}
1680
1681<ST_IN_SCRIPTING>"insteadof" {
1682    RETURN_TOKEN_WITH_IDENT(T_INSTEADOF);
1683}
1684
1685<ST_IN_SCRIPTING>"global" {
1686	RETURN_TOKEN_WITH_IDENT(T_GLOBAL);
1687}
1688
1689<ST_IN_SCRIPTING>"isset" {
1690	RETURN_TOKEN_WITH_IDENT(T_ISSET);
1691}
1692
1693<ST_IN_SCRIPTING>"empty" {
1694	RETURN_TOKEN_WITH_IDENT(T_EMPTY);
1695}
1696
1697<ST_IN_SCRIPTING>"__halt_compiler" {
1698	RETURN_TOKEN_WITH_IDENT(T_HALT_COMPILER);
1699}
1700
1701<ST_IN_SCRIPTING>"static" {
1702	RETURN_TOKEN_WITH_IDENT(T_STATIC);
1703}
1704
1705<ST_IN_SCRIPTING>"abstract" {
1706	RETURN_TOKEN_WITH_IDENT(T_ABSTRACT);
1707}
1708
1709<ST_IN_SCRIPTING>"final" {
1710	RETURN_TOKEN_WITH_IDENT(T_FINAL);
1711}
1712
1713<ST_IN_SCRIPTING>"private" {
1714	RETURN_TOKEN_WITH_IDENT(T_PRIVATE);
1715}
1716
1717<ST_IN_SCRIPTING>"protected" {
1718	RETURN_TOKEN_WITH_IDENT(T_PROTECTED);
1719}
1720
1721<ST_IN_SCRIPTING>"public" {
1722	RETURN_TOKEN_WITH_IDENT(T_PUBLIC);
1723}
1724
1725<ST_IN_SCRIPTING>"unset" {
1726	RETURN_TOKEN_WITH_IDENT(T_UNSET);
1727}
1728
1729<ST_IN_SCRIPTING>"=>" {
1730	RETURN_TOKEN(T_DOUBLE_ARROW);
1731}
1732
1733<ST_IN_SCRIPTING>"list" {
1734	RETURN_TOKEN_WITH_IDENT(T_LIST);
1735}
1736
1737<ST_IN_SCRIPTING>"array" {
1738	RETURN_TOKEN_WITH_IDENT(T_ARRAY);
1739}
1740
1741<ST_IN_SCRIPTING>"callable" {
1742	RETURN_TOKEN_WITH_IDENT(T_CALLABLE);
1743}
1744
1745<ST_IN_SCRIPTING>"++" {
1746	RETURN_TOKEN(T_INC);
1747}
1748
1749<ST_IN_SCRIPTING>"--" {
1750	RETURN_TOKEN(T_DEC);
1751}
1752
1753<ST_IN_SCRIPTING>"===" {
1754	RETURN_TOKEN(T_IS_IDENTICAL);
1755}
1756
1757<ST_IN_SCRIPTING>"!==" {
1758	RETURN_TOKEN(T_IS_NOT_IDENTICAL);
1759}
1760
1761<ST_IN_SCRIPTING>"==" {
1762	RETURN_TOKEN(T_IS_EQUAL);
1763}
1764
1765<ST_IN_SCRIPTING>"!="|"<>" {
1766	RETURN_TOKEN(T_IS_NOT_EQUAL);
1767}
1768
1769<ST_IN_SCRIPTING>"<=>" {
1770	RETURN_TOKEN(T_SPACESHIP);
1771}
1772
1773<ST_IN_SCRIPTING>"<=" {
1774	RETURN_TOKEN(T_IS_SMALLER_OR_EQUAL);
1775}
1776
1777<ST_IN_SCRIPTING>">=" {
1778	RETURN_TOKEN(T_IS_GREATER_OR_EQUAL);
1779}
1780
1781<ST_IN_SCRIPTING>"+=" {
1782	RETURN_TOKEN(T_PLUS_EQUAL);
1783}
1784
1785<ST_IN_SCRIPTING>"-=" {
1786	RETURN_TOKEN(T_MINUS_EQUAL);
1787}
1788
1789<ST_IN_SCRIPTING>"*=" {
1790	RETURN_TOKEN(T_MUL_EQUAL);
1791}
1792
1793<ST_IN_SCRIPTING>"*\*" {
1794	RETURN_TOKEN(T_POW);
1795}
1796
1797<ST_IN_SCRIPTING>"*\*=" {
1798	RETURN_TOKEN(T_POW_EQUAL);
1799}
1800
1801<ST_IN_SCRIPTING>"/=" {
1802	RETURN_TOKEN(T_DIV_EQUAL);
1803}
1804
1805<ST_IN_SCRIPTING>".=" {
1806	RETURN_TOKEN(T_CONCAT_EQUAL);
1807}
1808
1809<ST_IN_SCRIPTING>"%=" {
1810	RETURN_TOKEN(T_MOD_EQUAL);
1811}
1812
1813<ST_IN_SCRIPTING>"<<=" {
1814	RETURN_TOKEN(T_SL_EQUAL);
1815}
1816
1817<ST_IN_SCRIPTING>">>=" {
1818	RETURN_TOKEN(T_SR_EQUAL);
1819}
1820
1821<ST_IN_SCRIPTING>"&=" {
1822	RETURN_TOKEN(T_AND_EQUAL);
1823}
1824
1825<ST_IN_SCRIPTING>"|=" {
1826	RETURN_TOKEN(T_OR_EQUAL);
1827}
1828
1829<ST_IN_SCRIPTING>"^=" {
1830	RETURN_TOKEN(T_XOR_EQUAL);
1831}
1832
1833<ST_IN_SCRIPTING>"??=" {
1834	RETURN_TOKEN(T_COALESCE_EQUAL);
1835}
1836
1837<ST_IN_SCRIPTING>"||" {
1838	RETURN_TOKEN(T_BOOLEAN_OR);
1839}
1840
1841<ST_IN_SCRIPTING>"&&" {
1842	RETURN_TOKEN(T_BOOLEAN_AND);
1843}
1844
1845<ST_IN_SCRIPTING>"OR" {
1846	RETURN_TOKEN_WITH_IDENT(T_LOGICAL_OR);
1847}
1848
1849<ST_IN_SCRIPTING>"AND" {
1850	RETURN_TOKEN_WITH_IDENT(T_LOGICAL_AND);
1851}
1852
1853<ST_IN_SCRIPTING>"XOR" {
1854	RETURN_TOKEN_WITH_IDENT(T_LOGICAL_XOR);
1855}
1856
1857<ST_IN_SCRIPTING>"<<" {
1858	RETURN_TOKEN(T_SL);
1859}
1860
1861<ST_IN_SCRIPTING>">>" {
1862	RETURN_TOKEN(T_SR);
1863}
1864
1865<ST_IN_SCRIPTING>"]"|")" {
1866	/* Check that ] and ) match up properly with a preceding [ or ( */
1867	RETURN_EXIT_NESTING_TOKEN(yytext[0]);
1868}
1869
1870<ST_IN_SCRIPTING>"["|"(" {
1871	enter_nesting(yytext[0]);
1872	RETURN_TOKEN(yytext[0]);
1873}
1874
1875<ST_IN_SCRIPTING>{TOKENS} {
1876	RETURN_TOKEN(yytext[0]);
1877}
1878
1879
1880<ST_IN_SCRIPTING>"{" {
1881	yy_push_state(ST_IN_SCRIPTING);
1882	enter_nesting('{');
1883	RETURN_TOKEN('{');
1884}
1885
1886
1887<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
1888	yy_push_state(ST_LOOKING_FOR_VARNAME);
1889	enter_nesting('{');
1890	RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
1891}
1892
1893<ST_IN_SCRIPTING>"}" {
1894	RESET_DOC_COMMENT();
1895	if (!zend_stack_is_empty(&SCNG(state_stack))) {
1896		yy_pop_state();
1897	}
1898	RETURN_EXIT_NESTING_TOKEN('}');
1899}
1900
1901
1902<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
1903	yyless(yyleng - 1);
1904	yy_pop_state();
1905	yy_push_state(ST_IN_SCRIPTING);
1906	RETURN_TOKEN_WITH_STR(T_STRING_VARNAME, 0);
1907}
1908
1909
1910<ST_LOOKING_FOR_VARNAME>{ANY_CHAR} {
1911	yyless(0);
1912	yy_pop_state();
1913	yy_push_state(ST_IN_SCRIPTING);
1914	goto restart;
1915}
1916
1917<ST_IN_SCRIPTING>{BNUM} {
1918	/* The +/- 2 skips "0b" */
1919	size_t len = yyleng - 2;
1920	char *end, *bin = yytext + 2;
1921	zend_bool contains_underscores;
1922
1923	/* Skip any leading 0s */
1924	while (len > 0 && (*bin == '0' || *bin == '_')) {
1925		++bin;
1926		--len;
1927	}
1928
1929	contains_underscores = (memchr(bin, '_', len) != NULL);
1930
1931	if (contains_underscores) {
1932		bin = estrndup(bin, len);
1933		strip_underscores(bin, &len);
1934	}
1935
1936	if (len < SIZEOF_ZEND_LONG * 8) {
1937		if (len == 0) {
1938			ZVAL_LONG(zendlval, 0);
1939		} else {
1940			errno = 0;
1941			ZVAL_LONG(zendlval, ZEND_STRTOL(bin, &end, 2));
1942			ZEND_ASSERT(!errno && end == bin + len);
1943		}
1944		if (contains_underscores) {
1945			efree(bin);
1946		}
1947		RETURN_TOKEN_WITH_VAL(T_LNUMBER);
1948	} else {
1949		ZVAL_DOUBLE(zendlval, zend_bin_strtod(bin, (const char **)&end));
1950		/* errno isn't checked since we allow HUGE_VAL/INF overflow */
1951		ZEND_ASSERT(end == bin + len);
1952		if (contains_underscores) {
1953			efree(bin);
1954		}
1955		RETURN_TOKEN_WITH_VAL(T_DNUMBER);
1956	}
1957}
1958
1959<ST_IN_SCRIPTING>{LNUM} {
1960	size_t len = yyleng;
1961	char *end, *lnum = yytext;
1962	zend_bool is_octal = lnum[0] == '0';
1963	zend_bool contains_underscores = (memchr(lnum, '_', len) != NULL);
1964
1965	if (contains_underscores) {
1966		lnum = estrndup(lnum, len);
1967		strip_underscores(lnum, &len);
1968	}
1969
1970	/* Digits 8 and 9 are illegal in octal literals. */
1971	if (is_octal) {
1972		size_t i;
1973		for (i = 0; i < len; i++) {
1974			if (lnum[i] == '8' || lnum[i] == '9') {
1975				zend_throw_exception(zend_ce_parse_error, "Invalid numeric literal", 0);
1976				if (PARSER_MODE()) {
1977					if (contains_underscores) {
1978						efree(lnum);
1979					}
1980					ZVAL_UNDEF(zendlval);
1981					RETURN_TOKEN(T_ERROR);
1982				}
1983
1984				/* Continue in order to determine if this is T_LNUMBER or T_DNUMBER. */
1985				len = i;
1986				break;
1987			}
1988		}
1989	}
1990
1991
1992	if (len < MAX_LENGTH_OF_LONG - 1) { /* Won't overflow */
1993		errno = 0;
1994		/* base must be passed explicitly for correct parse error on Windows */
1995		ZVAL_LONG(zendlval, ZEND_STRTOL(lnum, &end, is_octal ? 8 : 10));
1996		ZEND_ASSERT(end == lnum + len);
1997	} else {
1998		errno = 0;
1999		ZVAL_LONG(zendlval, ZEND_STRTOL(lnum, &end, 0));
2000		if (errno == ERANGE) { /* Overflow */
2001			errno = 0;
2002			if (is_octal) { /* octal overflow */
2003				ZVAL_DOUBLE(zendlval, zend_oct_strtod(lnum, (const char **)&end));
2004			} else {
2005				ZVAL_DOUBLE(zendlval, zend_strtod(lnum, (const char **)&end));
2006			}
2007			ZEND_ASSERT(end == lnum + len);
2008			if (contains_underscores) {
2009				efree(lnum);
2010			}
2011			RETURN_TOKEN_WITH_VAL(T_DNUMBER);
2012		}
2013		ZEND_ASSERT(end == lnum + len);
2014	}
2015	ZEND_ASSERT(!errno);
2016	if (contains_underscores) {
2017		efree(lnum);
2018	}
2019	RETURN_TOKEN_WITH_VAL(T_LNUMBER);
2020}
2021
2022<ST_IN_SCRIPTING>{HNUM} {
2023	/* The +/- 2 skips "0x" */
2024	size_t len = yyleng - 2;
2025	char *end, *hex = yytext + 2;
2026	zend_bool contains_underscores;
2027
2028	/* Skip any leading 0s */
2029	while (len > 0 && (*hex == '0' || *hex == '_')) {
2030		++hex;
2031		--len;
2032	}
2033
2034	contains_underscores = (memchr(hex, '_', len) != NULL);
2035
2036	if (contains_underscores) {
2037		hex = estrndup(hex, len);
2038		strip_underscores(hex, &len);
2039	}
2040
2041	if (len < SIZEOF_ZEND_LONG * 2 || (len == SIZEOF_ZEND_LONG * 2 && *hex <= '7')) {
2042		if (len == 0) {
2043			ZVAL_LONG(zendlval, 0);
2044		} else {
2045			errno = 0;
2046			ZVAL_LONG(zendlval, ZEND_STRTOL(hex, &end, 16));
2047			ZEND_ASSERT(!errno && end == hex + len);
2048		}
2049		if (contains_underscores) {
2050			efree(hex);
2051		}
2052		RETURN_TOKEN_WITH_VAL(T_LNUMBER);
2053	} else {
2054		ZVAL_DOUBLE(zendlval, zend_hex_strtod(hex, (const char **)&end));
2055		/* errno isn't checked since we allow HUGE_VAL/INF overflow */
2056		ZEND_ASSERT(end == hex + len);
2057		if (contains_underscores) {
2058			efree(hex);
2059		}
2060		RETURN_TOKEN_WITH_VAL(T_DNUMBER);
2061	}
2062}
2063
2064<ST_VAR_OFFSET>[0]|([1-9][0-9]*) { /* Offset could be treated as a long */
2065	if (yyleng < MAX_LENGTH_OF_LONG - 1 || (yyleng == MAX_LENGTH_OF_LONG - 1 && strcmp(yytext, long_min_digits) < 0)) {
2066		char *end;
2067		errno = 0;
2068		ZVAL_LONG(zendlval, ZEND_STRTOL(yytext, &end, 10));
2069		if (errno == ERANGE) {
2070			goto string;
2071		}
2072		ZEND_ASSERT(end == yytext + yyleng);
2073	} else {
2074string:
2075		ZVAL_STRINGL(zendlval, yytext, yyleng);
2076	}
2077	RETURN_TOKEN_WITH_VAL(T_NUM_STRING);
2078}
2079
2080<ST_VAR_OFFSET>{LNUM}|{HNUM}|{BNUM} { /* Offset must be treated as a string */
2081	if (yyleng == 1) {
2082		ZVAL_INTERNED_STR(zendlval, ZSTR_CHAR((zend_uchar)*(yytext)));
2083	} else {
2084		ZVAL_STRINGL(zendlval, yytext, yyleng);
2085	}
2086	RETURN_TOKEN_WITH_VAL(T_NUM_STRING);
2087}
2088
2089<ST_IN_SCRIPTING>{DNUM}|{EXPONENT_DNUM} {
2090	const char *end;
2091	size_t len = yyleng;
2092	char *dnum = yytext;
2093	zend_bool contains_underscores = (memchr(dnum, '_', len) != NULL);
2094
2095	if (contains_underscores) {
2096		dnum = estrndup(dnum, len);
2097		strip_underscores(dnum, &len);
2098	}
2099
2100	ZVAL_DOUBLE(zendlval, zend_strtod(dnum, &end));
2101	/* errno isn't checked since we allow HUGE_VAL/INF overflow */
2102	ZEND_ASSERT(end == dnum + len);
2103	if (contains_underscores) {
2104		efree(dnum);
2105	}
2106	RETURN_TOKEN_WITH_VAL(T_DNUMBER);
2107}
2108
2109<ST_IN_SCRIPTING>"__CLASS__" {
2110	RETURN_TOKEN_WITH_IDENT(T_CLASS_C);
2111}
2112
2113<ST_IN_SCRIPTING>"__TRAIT__" {
2114	RETURN_TOKEN_WITH_IDENT(T_TRAIT_C);
2115}
2116
2117<ST_IN_SCRIPTING>"__FUNCTION__" {
2118	RETURN_TOKEN_WITH_IDENT(T_FUNC_C);
2119}
2120
2121<ST_IN_SCRIPTING>"__METHOD__" {
2122	RETURN_TOKEN_WITH_IDENT(T_METHOD_C);
2123}
2124
2125<ST_IN_SCRIPTING>"__LINE__" {
2126	RETURN_TOKEN_WITH_IDENT(T_LINE);
2127}
2128
2129<ST_IN_SCRIPTING>"__FILE__" {
2130	RETURN_TOKEN_WITH_IDENT(T_FILE);
2131}
2132
2133<ST_IN_SCRIPTING>"__DIR__" {
2134	RETURN_TOKEN_WITH_IDENT(T_DIR);
2135}
2136
2137<ST_IN_SCRIPTING>"__NAMESPACE__" {
2138	RETURN_TOKEN_WITH_IDENT(T_NS_C);
2139}
2140
2141<SHEBANG>"#!" .* {NEWLINE} {
2142	CG(zend_lineno)++;
2143	BEGIN(INITIAL);
2144	goto restart;
2145}
2146
2147<SHEBANG>{ANY_CHAR} {
2148	yyless(0);
2149	BEGIN(INITIAL);
2150	goto restart;
2151}
2152
2153<INITIAL>"<?=" {
2154	BEGIN(ST_IN_SCRIPTING);
2155	if (PARSER_MODE()) {
2156		/* We'll reject this as an identifier in zend_lex_tstring. */
2157		RETURN_TOKEN_WITH_IDENT(T_ECHO);
2158	}
2159	RETURN_TOKEN(T_OPEN_TAG_WITH_ECHO);
2160}
2161
2162
2163<INITIAL>"<?php"([ \t]|{NEWLINE}) {
2164	HANDLE_NEWLINE(yytext[yyleng-1]);
2165	BEGIN(ST_IN_SCRIPTING);
2166	RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
2167}
2168
2169<INITIAL>"<?php" {
2170	/* Allow <?php followed by end of file. */
2171	if (YYCURSOR == YYLIMIT) {
2172		BEGIN(ST_IN_SCRIPTING);
2173		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
2174	}
2175	/* Degenerate case: <?phpX is interpreted as <? phpX with short tags. */
2176	if (CG(short_tags)) {
2177		yyless(2);
2178		BEGIN(ST_IN_SCRIPTING);
2179		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
2180	}
2181	goto inline_char_handler;
2182}
2183
2184<INITIAL>"<?" {
2185	if (CG(short_tags)) {
2186		BEGIN(ST_IN_SCRIPTING);
2187		RETURN_OR_SKIP_TOKEN(T_OPEN_TAG);
2188	} else {
2189		goto inline_char_handler;
2190	}
2191}
2192
2193<INITIAL>{ANY_CHAR} {
2194	if (YYCURSOR > YYLIMIT) {
2195		RETURN_END_TOKEN;
2196	}
2197
2198inline_char_handler:
2199
2200	while (1) {
2201		YYCTYPE *ptr = memchr(YYCURSOR, '<', YYLIMIT - YYCURSOR);
2202
2203		YYCURSOR = ptr ? ptr + 1 : YYLIMIT;
2204
2205		if (YYCURSOR >= YYLIMIT) {
2206			break;
2207		}
2208
2209		if (*YYCURSOR == '?') {
2210			if (CG(short_tags) /* <? */
2211				|| (*(YYCURSOR + 1) == '=') /* <?= */
2212				|| (!strncasecmp((char*)YYCURSOR + 1, "php", 3) && /* <?php[ \t\r\n] */
2213					(YYCURSOR + 4 == YYLIMIT ||
2214					YYCURSOR[4] == ' ' || YYCURSOR[4] == '\t' ||
2215					YYCURSOR[4] == '\n' || YYCURSOR[4] == '\r'))
2216			) {
2217				YYCURSOR--;
2218				break;
2219			}
2220		}
2221	}
2222
2223	yyleng = YYCURSOR - SCNG(yy_text);
2224
2225	if (SCNG(output_filter)) {
2226		size_t readsize;
2227		char *s = NULL;
2228		size_t sz = 0;
2229		// TODO: avoid reallocation ???
2230		readsize = SCNG(output_filter)((unsigned char **)&s, &sz, (unsigned char *)yytext, (size_t)yyleng);
2231		ZVAL_STRINGL(zendlval, s, sz);
2232		efree(s);
2233		if (readsize < yyleng) {
2234			yyless(readsize);
2235		}
2236	} else if (yyleng == 1) {
2237		ZVAL_INTERNED_STR(zendlval, ZSTR_CHAR((zend_uchar)*yytext));
2238	} else {
2239		ZVAL_STRINGL(zendlval, yytext, yyleng);
2240	}
2241	HANDLE_NEWLINES(yytext, yyleng);
2242	RETURN_TOKEN_WITH_VAL(T_INLINE_HTML);
2243}
2244
2245
2246/* Make sure a label character follows "->" or "?->", otherwise there is no property
2247 * and "->"/"?->" will be taken literally
2248 */
2249<ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL}"->"[a-zA-Z_\x80-\xff] {
2250	yyless(yyleng - 3);
2251	yy_push_state(ST_LOOKING_FOR_PROPERTY);
2252	RETURN_TOKEN_WITH_STR(T_VARIABLE, 1);
2253}
2254
2255<ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL}"?->"[a-zA-Z_\x80-\xff] {
2256	yyless(yyleng - 4);
2257	yy_push_state(ST_LOOKING_FOR_PROPERTY);
2258	RETURN_TOKEN_WITH_STR(T_VARIABLE, 1);
2259}
2260
2261/* A [ always designates a variable offset, regardless of what follows
2262 */
2263<ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL}"[" {
2264	yyless(yyleng - 1);
2265	yy_push_state(ST_VAR_OFFSET);
2266	RETURN_TOKEN_WITH_STR(T_VARIABLE, 1);
2267}
2268
2269<ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE,ST_VAR_OFFSET>"$"{LABEL} {
2270	RETURN_TOKEN_WITH_STR(T_VARIABLE, 1);
2271}
2272
2273<ST_VAR_OFFSET>"]" {
2274	yy_pop_state();
2275	RETURN_TOKEN(']');
2276}
2277
2278<ST_VAR_OFFSET>{TOKENS}|[[(){}"`] {
2279	/* Only '[' or '-' can be valid, but returning other tokens will allow a more explicit parse error */
2280	RETURN_TOKEN(yytext[0]);
2281}
2282
2283<ST_VAR_OFFSET>[ \n\r\t\\'#] {
2284	/* Invalid rule to return a more explicit parse error with proper line number */
2285	yyless(0);
2286	yy_pop_state();
2287	ZVAL_NULL(zendlval);
2288	RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
2289}
2290
2291<ST_IN_SCRIPTING>"namespace"("\\"{LABEL})+ {
2292	RETURN_TOKEN_WITH_STR(T_NAME_RELATIVE, sizeof("namespace\\") - 1);
2293}
2294
2295<ST_IN_SCRIPTING>{LABEL}("\\"{LABEL})+ {
2296	RETURN_TOKEN_WITH_STR(T_NAME_QUALIFIED, 0);
2297}
2298
2299<ST_IN_SCRIPTING>"\\"{LABEL}("\\"{LABEL})* {
2300	RETURN_TOKEN_WITH_STR(T_NAME_FULLY_QUALIFIED, 1);
2301}
2302
2303<ST_IN_SCRIPTING>"\\" {
2304	RETURN_TOKEN(T_NS_SEPARATOR);
2305}
2306
2307<ST_IN_SCRIPTING,ST_VAR_OFFSET>{LABEL} {
2308	RETURN_TOKEN_WITH_STR(T_STRING, 0);
2309}
2310
2311
2312<ST_IN_SCRIPTING>"#"|"//" {
2313	while (YYCURSOR < YYLIMIT) {
2314		switch (*YYCURSOR++) {
2315			case '\r':
2316			case '\n':
2317				YYCURSOR--;
2318				break;
2319			case '?':
2320				if (*YYCURSOR == '>') {
2321					YYCURSOR--;
2322					break;
2323				}
2324				/* fall through */
2325			default:
2326				continue;
2327		}
2328
2329		break;
2330	}
2331
2332	yyleng = YYCURSOR - SCNG(yy_text);
2333	RETURN_OR_SKIP_TOKEN(T_COMMENT);
2334}
2335
2336<ST_IN_SCRIPTING>"/*"|"/**"{WHITESPACE} {
2337	int doc_com;
2338
2339	if (yyleng > 2) {
2340		doc_com = 1;
2341		RESET_DOC_COMMENT();
2342	} else {
2343		doc_com = 0;
2344	}
2345
2346	while (YYCURSOR < YYLIMIT) {
2347		if (*YYCURSOR++ == '*' && *YYCURSOR == '/') {
2348			break;
2349		}
2350	}
2351
2352	if (YYCURSOR < YYLIMIT) {
2353		YYCURSOR++;
2354	} else {
2355		zend_throw_exception_ex(zend_ce_parse_error, 0, "Unterminated comment starting line %d", CG(zend_lineno));
2356		if (PARSER_MODE()) {
2357			RETURN_TOKEN(T_ERROR);
2358		}
2359	}
2360
2361	yyleng = YYCURSOR - SCNG(yy_text);
2362	HANDLE_NEWLINES(yytext, yyleng);
2363
2364	if (doc_com) {
2365		CG(doc_comment) = zend_string_init(yytext, yyleng, 0);
2366		RETURN_OR_SKIP_TOKEN(T_DOC_COMMENT);
2367	}
2368
2369	RETURN_OR_SKIP_TOKEN(T_COMMENT);
2370}
2371
2372<ST_IN_SCRIPTING>"?>"{NEWLINE}? {
2373	BEGIN(INITIAL);
2374	if (yytext[yyleng-1] != '>') {
2375		CG(increment_lineno) = 1;
2376	}
2377	if (PARSER_MODE()) {
2378		RETURN_TOKEN(';');  /* implicit ';' at php-end tag */
2379	}
2380	RETURN_TOKEN(T_CLOSE_TAG);
2381}
2382
2383
2384<ST_IN_SCRIPTING>b?['] {
2385	register char *s, *t;
2386	char *end;
2387	int bprefix = (yytext[0] != '\'') ? 1 : 0;
2388
2389	while (1) {
2390		if (YYCURSOR < YYLIMIT) {
2391			if (*YYCURSOR == '\'') {
2392				YYCURSOR++;
2393				yyleng = YYCURSOR - SCNG(yy_text);
2394
2395				break;
2396			} else if (*YYCURSOR++ == '\\' && YYCURSOR < YYLIMIT) {
2397				YYCURSOR++;
2398			}
2399		} else {
2400			yyleng = YYLIMIT - SCNG(yy_text);
2401
2402			/* Unclosed single quotes; treat similar to double quotes, but without a separate token
2403			 * for ' (unrecognized by parser), instead of old flex fallback to "Unexpected character..."
2404			 * rule, which continued in ST_IN_SCRIPTING state after the quote */
2405			ZVAL_NULL(zendlval);
2406			RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
2407		}
2408	}
2409
2410	if (yyleng-bprefix-2 <= 1) {
2411		if (yyleng-bprefix-2 < 1) {
2412			ZVAL_EMPTY_STRING(zendlval);
2413		} else {
2414			zend_uchar c = (zend_uchar)*(yytext+bprefix+1);
2415			if (c == '\n' || c == '\r') {
2416				CG(zend_lineno)++;
2417			}
2418			ZVAL_INTERNED_STR(zendlval, ZSTR_CHAR(c));
2419		}
2420		goto skip_escape_conversion;
2421	}
2422	ZVAL_STRINGL(zendlval, yytext+bprefix+1, yyleng-bprefix-2);
2423
2424	/* convert escape sequences */
2425	s = Z_STRVAL_P(zendlval);
2426	end = s+Z_STRLEN_P(zendlval);
2427	while (1) {
2428		if (UNEXPECTED(*s=='\\')) {
2429			break;
2430		}
2431		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
2432			CG(zend_lineno)++;
2433		}
2434		s++;
2435		if (s == end) {
2436			goto skip_escape_conversion;
2437		}
2438	}
2439
2440	t = s;
2441	while (s<end) {
2442		if (*s=='\\') {
2443			s++;
2444			if (*s == '\\' || *s == '\'') {
2445				*t++ = *s;
2446			} else {
2447				*t++ = '\\';
2448				*t++ = *s;
2449			}
2450		} else {
2451			*t++ = *s;
2452		}
2453		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
2454			CG(zend_lineno)++;
2455		}
2456		s++;
2457	}
2458	*t = 0;
2459	Z_STRLEN_P(zendlval) = t - Z_STRVAL_P(zendlval);
2460
2461skip_escape_conversion:
2462	if (SCNG(output_filter)) {
2463		size_t sz = 0;
2464		char *str = NULL;
2465		zend_string *new_str;
2466		s = Z_STRVAL_P(zendlval);
2467		// TODO: avoid reallocation ???
2468		SCNG(output_filter)((unsigned char **)&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
2469		new_str = zend_string_init(str, sz, 0);
2470		if (str != s) {
2471			efree(str);
2472		}
2473		zend_string_release_ex(Z_STR_P(zendlval), 0);
2474		ZVAL_STR(zendlval, new_str);
2475	}
2476	RETURN_TOKEN_WITH_VAL(T_CONSTANT_ENCAPSED_STRING);
2477}
2478
2479
2480<ST_IN_SCRIPTING>b?["] {
2481	int bprefix = (yytext[0] != '"') ? 1 : 0;
2482
2483	while (YYCURSOR < YYLIMIT) {
2484		switch (*YYCURSOR++) {
2485			case '"':
2486				yyleng = YYCURSOR - SCNG(yy_text);
2487				if (EXPECTED(zend_scan_escape_string(zendlval, yytext+bprefix+1, yyleng-bprefix-2, '"') == SUCCESS)
2488				 || !PARSER_MODE()) {
2489					RETURN_TOKEN_WITH_VAL(T_CONSTANT_ENCAPSED_STRING);
2490				} else {
2491					RETURN_TOKEN(T_ERROR);
2492				}
2493			case '$':
2494				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2495					break;
2496				}
2497				continue;
2498			case '{':
2499				if (*YYCURSOR == '$') {
2500					break;
2501				}
2502				continue;
2503			case '\\':
2504				if (YYCURSOR < YYLIMIT) {
2505					YYCURSOR++;
2506				}
2507				/* fall through */
2508			default:
2509				continue;
2510		}
2511
2512		YYCURSOR--;
2513		break;
2514	}
2515
2516	/* Remember how much was scanned to save rescanning */
2517	SET_DOUBLE_QUOTES_SCANNED_LENGTH(YYCURSOR - SCNG(yy_text) - yyleng);
2518
2519	YYCURSOR = SCNG(yy_text) + yyleng;
2520
2521	BEGIN(ST_DOUBLE_QUOTES);
2522	RETURN_TOKEN('"');
2523}
2524
2525
2526<ST_IN_SCRIPTING>b?"<<<"{TABS_AND_SPACES}({LABEL}|([']{LABEL}['])|(["]{LABEL}["])){NEWLINE} {
2527	char *s;
2528	unsigned char *saved_cursor;
2529	int bprefix = (yytext[0] != '<') ? 1 : 0, spacing = 0, indentation = 0;
2530	zend_heredoc_label *heredoc_label = emalloc(sizeof(zend_heredoc_label));
2531	zend_bool is_heredoc = 1;
2532
2533	CG(zend_lineno)++;
2534	heredoc_label->length = yyleng-bprefix-3-1-(yytext[yyleng-2]=='\r'?1:0);
2535	s = yytext+bprefix+3;
2536	while ((*s == ' ') || (*s == '\t')) {
2537		s++;
2538		heredoc_label->length--;
2539	}
2540
2541	if (*s == '\'') {
2542		s++;
2543		heredoc_label->length -= 2;
2544		is_heredoc = 0;
2545
2546		BEGIN(ST_NOWDOC);
2547	} else {
2548		if (*s == '"') {
2549			s++;
2550			heredoc_label->length -= 2;
2551		}
2552
2553		BEGIN(ST_HEREDOC);
2554	}
2555
2556	heredoc_label->label = estrndup(s, heredoc_label->length);
2557	heredoc_label->indentation_uses_spaces = 0;
2558	heredoc_label->indentation = 0;
2559	saved_cursor = YYCURSOR;
2560
2561	zend_ptr_stack_push(&SCNG(heredoc_label_stack), (void *) heredoc_label);
2562
2563	while (YYCURSOR < YYLIMIT && (*YYCURSOR == ' ' || *YYCURSOR == '\t')) {
2564		if (*YYCURSOR == '\t') {
2565			spacing |= HEREDOC_USING_TABS;
2566		} else {
2567			spacing |= HEREDOC_USING_SPACES;
2568		}
2569		++YYCURSOR;
2570		++indentation;
2571	}
2572
2573	if (YYCURSOR == YYLIMIT) {
2574		YYCURSOR = saved_cursor;
2575		RETURN_TOKEN(T_START_HEREDOC);
2576	}
2577
2578	/* Check for ending label on the next line */
2579	if (heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, s, heredoc_label->length)) {
2580		if (!IS_LABEL_SUCCESSOR(YYCURSOR[heredoc_label->length])) {
2581			if (spacing == (HEREDOC_USING_SPACES | HEREDOC_USING_TABS)) {
2582				zend_throw_exception(zend_ce_parse_error, "Invalid indentation - tabs and spaces cannot be mixed", 0);
2583				if (PARSER_MODE()) {
2584					RETURN_TOKEN(T_ERROR);
2585				}
2586			}
2587
2588			YYCURSOR = saved_cursor;
2589			heredoc_label->indentation = indentation;
2590
2591			BEGIN(ST_END_HEREDOC);
2592			RETURN_TOKEN(T_START_HEREDOC);
2593		}
2594	}
2595
2596	YYCURSOR = saved_cursor;
2597
2598	if (is_heredoc && !SCNG(heredoc_scan_ahead)) {
2599		zend_lex_state current_state;
2600		zend_string *saved_doc_comment = CG(doc_comment);
2601		int heredoc_nesting_level = 1;
2602		int first_token = 0;
2603		int error = 0;
2604
2605		zend_save_lexical_state(&current_state);
2606
2607		SCNG(heredoc_scan_ahead) = 1;
2608		SCNG(heredoc_indentation) = 0;
2609		SCNG(heredoc_indentation_uses_spaces) = 0;
2610		LANG_SCNG(on_event) = NULL;
2611		CG(doc_comment) = NULL;
2612
2613		zend_ptr_stack_reverse_apply(&current_state.heredoc_label_stack, copy_heredoc_label_stack);
2614
2615		zend_exception_save();
2616		while (heredoc_nesting_level) {
2617			zval zv;
2618			int retval;
2619
2620			ZVAL_UNDEF(&zv);
2621			retval = lex_scan(&zv, NULL);
2622			zval_ptr_dtor_nogc(&zv);
2623
2624			if (EG(exception)) {
2625				zend_clear_exception();
2626				break;
2627			}
2628
2629			if (!first_token) {
2630				first_token = retval;
2631			}
2632
2633			switch (retval) {
2634				case T_START_HEREDOC:
2635					++heredoc_nesting_level;
2636					break;
2637				case T_END_HEREDOC:
2638					--heredoc_nesting_level;
2639					break;
2640				case END:
2641					heredoc_nesting_level = 0;
2642			}
2643		}
2644		zend_exception_restore();
2645
2646		if (
2647		    (first_token == T_VARIABLE
2648		     || first_token == T_DOLLAR_OPEN_CURLY_BRACES
2649		     || first_token == T_CURLY_OPEN
2650		    ) && SCNG(heredoc_indentation)) {
2651			zend_throw_exception_ex(zend_ce_parse_error, 0, "Invalid body indentation level (expecting an indentation level of at least %d)", SCNG(heredoc_indentation));
2652			error = 1;
2653		}
2654
2655		heredoc_label->indentation = SCNG(heredoc_indentation);
2656		heredoc_label->indentation_uses_spaces = SCNG(heredoc_indentation_uses_spaces);
2657
2658		zend_restore_lexical_state(&current_state);
2659		SCNG(heredoc_scan_ahead) = 0;
2660		CG(increment_lineno) = 0;
2661		CG(doc_comment) = saved_doc_comment;
2662
2663		if (PARSER_MODE() && error) {
2664			RETURN_TOKEN(T_ERROR);
2665		}
2666	}
2667
2668	RETURN_TOKEN(T_START_HEREDOC);
2669}
2670
2671
2672<ST_IN_SCRIPTING>[`] {
2673	BEGIN(ST_BACKQUOTE);
2674	RETURN_TOKEN('`');
2675}
2676
2677
2678<ST_END_HEREDOC>{ANY_CHAR} {
2679	zend_heredoc_label *heredoc_label = zend_ptr_stack_pop(&SCNG(heredoc_label_stack));
2680
2681	yyleng = heredoc_label->indentation + heredoc_label->length;
2682	YYCURSOR += yyleng - 1;
2683
2684	heredoc_label_dtor(heredoc_label);
2685	efree(heredoc_label);
2686
2687	BEGIN(ST_IN_SCRIPTING);
2688	RETURN_TOKEN(T_END_HEREDOC);
2689}
2690
2691
2692<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"{$" {
2693	yy_push_state(ST_IN_SCRIPTING);
2694	yyless(1);
2695	enter_nesting('{');
2696	RETURN_TOKEN(T_CURLY_OPEN);
2697}
2698
2699
2700<ST_DOUBLE_QUOTES>["] {
2701	BEGIN(ST_IN_SCRIPTING);
2702	RETURN_TOKEN('"');
2703}
2704
2705<ST_BACKQUOTE>[`] {
2706	BEGIN(ST_IN_SCRIPTING);
2707	RETURN_TOKEN('`');
2708}
2709
2710
2711<ST_DOUBLE_QUOTES>{ANY_CHAR} {
2712	if (GET_DOUBLE_QUOTES_SCANNED_LENGTH()) {
2713		YYCURSOR += GET_DOUBLE_QUOTES_SCANNED_LENGTH() - 1;
2714		SET_DOUBLE_QUOTES_SCANNED_LENGTH(0);
2715
2716		goto double_quotes_scan_done;
2717	}
2718
2719	if (YYCURSOR > YYLIMIT) {
2720		RETURN_END_TOKEN;
2721	}
2722	if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) {
2723		YYCURSOR++;
2724	}
2725
2726	while (YYCURSOR < YYLIMIT) {
2727		switch (*YYCURSOR++) {
2728			case '"':
2729				break;
2730			case '$':
2731				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2732					break;
2733				}
2734				continue;
2735			case '{':
2736				if (*YYCURSOR == '$') {
2737					break;
2738				}
2739				continue;
2740			case '\\':
2741				if (YYCURSOR < YYLIMIT) {
2742					YYCURSOR++;
2743				}
2744				/* fall through */
2745			default:
2746				continue;
2747		}
2748
2749		YYCURSOR--;
2750		break;
2751	}
2752
2753double_quotes_scan_done:
2754	yyleng = YYCURSOR - SCNG(yy_text);
2755
2756	if (EXPECTED(zend_scan_escape_string(zendlval, yytext, yyleng, '"') == SUCCESS)
2757	 || !PARSER_MODE()) {
2758		RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
2759	} else {
2760		RETURN_TOKEN(T_ERROR);
2761	}
2762}
2763
2764
2765<ST_BACKQUOTE>{ANY_CHAR} {
2766	if (YYCURSOR > YYLIMIT) {
2767		RETURN_END_TOKEN;
2768	}
2769	if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) {
2770		YYCURSOR++;
2771	}
2772
2773	while (YYCURSOR < YYLIMIT) {
2774		switch (*YYCURSOR++) {
2775			case '`':
2776				break;
2777			case '$':
2778				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2779					break;
2780				}
2781				continue;
2782			case '{':
2783				if (*YYCURSOR == '$') {
2784					break;
2785				}
2786				continue;
2787			case '\\':
2788				if (YYCURSOR < YYLIMIT) {
2789					YYCURSOR++;
2790				}
2791				/* fall through */
2792			default:
2793				continue;
2794		}
2795
2796		YYCURSOR--;
2797		break;
2798	}
2799
2800	yyleng = YYCURSOR - SCNG(yy_text);
2801
2802	if (EXPECTED(zend_scan_escape_string(zendlval, yytext, yyleng, '`') == SUCCESS)
2803	 || !PARSER_MODE()) {
2804		RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
2805	} else {
2806		RETURN_TOKEN(T_ERROR);
2807	}
2808}
2809
2810
2811<ST_HEREDOC>{ANY_CHAR} {
2812	zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack));
2813	int newline = 0, indentation = 0, spacing = 0;
2814
2815	if (YYCURSOR > YYLIMIT) {
2816		RETURN_END_TOKEN;
2817	}
2818
2819	YYCURSOR--;
2820
2821	while (YYCURSOR < YYLIMIT) {
2822		switch (*YYCURSOR++) {
2823			case '\r':
2824				if (*YYCURSOR == '\n') {
2825					YYCURSOR++;
2826				}
2827				/* fall through */
2828			case '\n':
2829				indentation = spacing = 0;
2830
2831				while (YYCURSOR < YYLIMIT && (*YYCURSOR == ' ' || *YYCURSOR == '\t')) {
2832					if (*YYCURSOR == '\t') {
2833						spacing |= HEREDOC_USING_TABS;
2834					} else {
2835						spacing |= HEREDOC_USING_SPACES;
2836					}
2837					++YYCURSOR;
2838					++indentation;
2839				}
2840
2841				if (YYCURSOR == YYLIMIT) {
2842					yyleng = YYCURSOR - SCNG(yy_text);
2843					HANDLE_NEWLINES(yytext, yyleng);
2844					ZVAL_NULL(zendlval);
2845					RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
2846				}
2847
2848				/* Check for ending label on the next line */
2849				if (IS_LABEL_START(*YYCURSOR) && heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, heredoc_label->label, heredoc_label->length)) {
2850					if (IS_LABEL_SUCCESSOR(YYCURSOR[heredoc_label->length])) {
2851						continue;
2852					}
2853
2854					if (spacing == (HEREDOC_USING_SPACES | HEREDOC_USING_TABS)) {
2855						zend_throw_exception(zend_ce_parse_error, "Invalid indentation - tabs and spaces cannot be mixed", 0);
2856						if (PARSER_MODE()) {
2857							RETURN_TOKEN(T_ERROR);
2858						}
2859					}
2860
2861					/* newline before label will be subtracted from returned text, but
2862					 * yyleng/yytext will include it, for zend_highlight/strip, tokenizer, etc. */
2863					if (YYCURSOR[-indentation - 2] == '\r' && YYCURSOR[-indentation - 1] == '\n') {
2864						newline = 2; /* Windows newline */
2865					} else {
2866						newline = 1;
2867					}
2868
2869					CG(increment_lineno) = 1; /* For newline before label */
2870
2871					if (SCNG(heredoc_scan_ahead)) {
2872						SCNG(heredoc_indentation) = indentation;
2873						SCNG(heredoc_indentation_uses_spaces) = (spacing == HEREDOC_USING_SPACES);
2874					} else {
2875						YYCURSOR -= indentation;
2876					}
2877
2878					BEGIN(ST_END_HEREDOC);
2879
2880					goto heredoc_scan_done;
2881				}
2882				continue;
2883			case '$':
2884				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2885					break;
2886				}
2887				continue;
2888			case '{':
2889				if (*YYCURSOR == '$') {
2890					break;
2891				}
2892				continue;
2893			case '\\':
2894				if (YYCURSOR < YYLIMIT && *YYCURSOR != '\n' && *YYCURSOR != '\r') {
2895					YYCURSOR++;
2896				}
2897				/* fall through */
2898			default:
2899				continue;
2900		}
2901
2902		YYCURSOR--;
2903		break;
2904	}
2905
2906heredoc_scan_done:
2907
2908	yyleng = YYCURSOR - SCNG(yy_text);
2909	ZVAL_STRINGL(zendlval, yytext, yyleng - newline);
2910
2911	if (!SCNG(heredoc_scan_ahead) && !EG(exception) && PARSER_MODE()) {
2912		zend_bool newline_at_start = *(yytext - 1) == '\n' || *(yytext - 1) == '\r';
2913		zend_string *copy = Z_STR_P(zendlval);
2914
2915		if (!strip_multiline_string_indentation(
2916				zendlval, heredoc_label->indentation, heredoc_label->indentation_uses_spaces,
2917				newline_at_start, newline != 0)) {
2918			RETURN_TOKEN(T_ERROR);
2919		}
2920
2921		if (UNEXPECTED(zend_scan_escape_string(zendlval, ZSTR_VAL(copy), ZSTR_LEN(copy), 0) != SUCCESS)) {
2922			zend_string_efree(copy);
2923			RETURN_TOKEN(T_ERROR);
2924		}
2925
2926		zend_string_efree(copy);
2927	} else {
2928		HANDLE_NEWLINES(yytext, yyleng - newline);
2929	}
2930
2931	RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
2932}
2933
2934
2935<ST_NOWDOC>{ANY_CHAR} {
2936	zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack));
2937	int newline = 0, indentation = 0, spacing = -1;
2938
2939	if (YYCURSOR > YYLIMIT) {
2940		RETURN_END_TOKEN;
2941	}
2942
2943	YYCURSOR--;
2944
2945	while (YYCURSOR < YYLIMIT) {
2946		switch (*YYCURSOR++) {
2947			case '\r':
2948				if (*YYCURSOR == '\n') {
2949					YYCURSOR++;
2950				}
2951				/* fall through */
2952			case '\n':
2953				indentation = spacing = 0;
2954
2955				while (YYCURSOR < YYLIMIT && (*YYCURSOR == ' ' || *YYCURSOR == '\t')) {
2956					if (*YYCURSOR == '\t') {
2957						spacing |= HEREDOC_USING_TABS;
2958					} else {
2959						spacing |= HEREDOC_USING_SPACES;
2960					}
2961					++YYCURSOR;
2962					++indentation;
2963				}
2964
2965				if (YYCURSOR == YYLIMIT) {
2966					yyleng = YYCURSOR - SCNG(yy_text);
2967					HANDLE_NEWLINES(yytext, yyleng);
2968					ZVAL_NULL(zendlval);
2969					RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
2970				}
2971
2972				/* Check for ending label on the next line */
2973				if (IS_LABEL_START(*YYCURSOR) && heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, heredoc_label->label, heredoc_label->length)) {
2974					if (IS_LABEL_SUCCESSOR(YYCURSOR[heredoc_label->length])) {
2975						continue;
2976					}
2977
2978					if (spacing == (HEREDOC_USING_SPACES | HEREDOC_USING_TABS)) {
2979						zend_throw_exception(zend_ce_parse_error, "Invalid indentation - tabs and spaces cannot be mixed", 0);
2980						if (PARSER_MODE()) {
2981							RETURN_TOKEN(T_ERROR);
2982						}
2983					}
2984
2985					/* newline before label will be subtracted from returned text, but
2986					 * yyleng/yytext will include it, for zend_highlight/strip, tokenizer, etc. */
2987					if (YYCURSOR[-indentation - 2] == '\r' && YYCURSOR[-indentation - 1] == '\n') {
2988						newline = 2; /* Windows newline */
2989					} else {
2990						newline = 1;
2991					}
2992
2993					CG(increment_lineno) = 1; /* For newline before label */
2994
2995					YYCURSOR -= indentation;
2996					heredoc_label->indentation = indentation;
2997
2998					BEGIN(ST_END_HEREDOC);
2999
3000					goto nowdoc_scan_done;
3001				}
3002				/* fall through */
3003			default:
3004				continue;
3005		}
3006	}
3007
3008nowdoc_scan_done:
3009	yyleng = YYCURSOR - SCNG(yy_text);
3010	ZVAL_STRINGL(zendlval, yytext, yyleng - newline);
3011
3012	if (!EG(exception) && spacing != -1 && PARSER_MODE()) {
3013		zend_bool newline_at_start = *(yytext - 1) == '\n' || *(yytext - 1) == '\r';
3014		if (!strip_multiline_string_indentation(
3015				zendlval, indentation, spacing == HEREDOC_USING_SPACES,
3016				newline_at_start, newline != 0)) {
3017			RETURN_TOKEN(T_ERROR);
3018		}
3019	}
3020
3021	HANDLE_NEWLINES(yytext, yyleng - newline);
3022	RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
3023}
3024
3025
3026<ST_IN_SCRIPTING,ST_VAR_OFFSET>{ANY_CHAR} {
3027	if (YYCURSOR > YYLIMIT) {
3028		RETURN_END_TOKEN;
3029	}
3030
3031	RETURN_TOKEN(T_BAD_CHARACTER);
3032}
3033
3034*/
3035
3036emit_token_with_str:
3037	zend_copy_value(zendlval, (yytext + offset), (yyleng - offset));
3038
3039emit_token_with_val:
3040	if (PARSER_MODE()) {
3041		ZEND_ASSERT(Z_TYPE_P(zendlval) != IS_UNDEF);
3042		elem->ast = zend_ast_create_zval_with_lineno(zendlval, start_line);
3043	}
3044
3045emit_token:
3046	if (SCNG(on_event)) {
3047		SCNG(on_event)(ON_TOKEN, token, start_line, yytext, yyleng, SCNG(on_event_context));
3048	}
3049	return token;
3050
3051emit_token_with_ident:
3052	if (PARSER_MODE()) {
3053		elem->ident.offset = SCNG(yy_text) - SCNG(yy_start);
3054		elem->ident.len = SCNG(yy_leng);
3055	}
3056	if (SCNG(on_event)) {
3057		SCNG(on_event)(ON_TOKEN, token, start_line, yytext, yyleng, SCNG(on_event_context));
3058	}
3059	return token;
3060
3061return_whitespace:
3062	HANDLE_NEWLINES(yytext, yyleng);
3063	if (SCNG(on_event)) {
3064		SCNG(on_event)(ON_TOKEN, T_WHITESPACE, start_line, yytext, yyleng, SCNG(on_event_context));
3065	}
3066	if (PARSER_MODE()) {
3067		start_line = CG(zend_lineno);
3068		goto restart;
3069	} else {
3070		return T_WHITESPACE;
3071	}
3072
3073skip_token:
3074	if (SCNG(on_event)) {
3075		SCNG(on_event)(ON_TOKEN, token, start_line, yytext, yyleng, SCNG(on_event_context));
3076	}
3077	start_line = CG(zend_lineno);
3078	goto restart;
3079}
3080